From 47e1905561290f2617ba894a7e9ad50129705d7f Mon Sep 17 00:00:00 2001
From: Muhammad Afaq Shuaib <78806673+AfaqShuaib09@users.noreply.github.com>
Date: Wed, 11 Jan 2023 16:20:33 +0500
Subject: [PATCH 01/14] fix: override clean_html function for anchor tag to
 save target atrribute (#3752)

Co-authored-by: afaq.shuaib <afaq.shuaib09@arbisoft.com>
---
 .../apps/course_metadata/tests/test_utils.py  |  6 +++++
 .../apps/course_metadata/utils.py             | 26 ++++++++++++-------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index bb178aa8bf6..603d57bc64b 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -619,6 +619,12 @@ class UtilsTests(TestCase):
         # Make sure we treat incoming text as HTML, not markdown
         ('Bare Text\nSame Para\n\nNew Para', '<p>Bare Text Same Para New Para</p>'),
 
+        # Make sure to add target attributes to anchor tags if they are in attributes list
+        # pylint: disable=line-too-long
+        ('<p>please visit this <a title="less go" href="https://google.com" target="_blank" rel="noopener">link</a></p>', '<p>please visit this <a href="https://google.com" rel="noopener" target="_blank" title="less go">link</a></p>'),
+        # pylint: disable=line-too-long
+        ('<a href="https://yahoo.com" target="_blank" rel="noopener">link</a>', '<p><a href="https://yahoo.com" rel="noopener" target="_blank">link</a></p>'),
+
         # And make sure we strip what we should
         ('<p class="float">Class</p>', '<p>Class</p>'),
         ('<p style="float">Inline Style</p>', '<p>Inline Style</p>'),
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index ea72a244cb0..ab4b7bb3bc7 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -646,23 +646,31 @@ class HTML2TextWithLangSpans(html2text.HTML2Text):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.ignore_links = True
         self.in_lang_span = False
         self.images_with_size = True
 
     def handle_tag(self, tag, attrs, start):
         super().handle_tag(tag, attrs, start)
         if tag == 'span':
-            if attrs:
-                attr_dict = dict(attrs)
-                if start and 'lang' in attr_dict:
-                    lang = attr_dict['lang']
-                    self.outtextf(f'<span lang="{lang}">')
-                    self.in_lang_span = True
-            if not start:
-                if self.in_lang_span:
-                    self.outtextf('</span>')
+            if attrs and start and 'lang' in dict(attrs):
+                self.outtextf(f'<span lang="{dict(attrs)["lang"]}">')
+                self.in_lang_span = True
+            if not start and self.in_lang_span:
+                self.outtextf('</span>')
                 self.in_lang_span = False
 
+        if tag == 'a':
+            # override the default behavior of html2text to include all attributes from attr_dict for <a> tags
+            # because by default it only includes the href and title attributes
+            if attrs and start and 'href' in dict(attrs):
+                self.outtextf('<a')
+                for attr, value in dict(attrs).items():
+                    self.outtextf(f' {attr}="{value}"')
+                self.outtextf('>')
+            if not start:
+                self.outtextf('</a>')
+
 
 def clean_html(content):
     """Cleans HTML from a string.

From d7a1b4059ff8b93411736b599784b744590a260f Mon Sep 17 00:00:00 2001
From: ansabgillani <syedansab500@gmail.com>
Date: Mon, 9 Jan 2023 21:11:51 +0500
Subject: [PATCH 02/14] feat: add external source identifier config model for
 programs and courses

---
 course_discovery/apps/api/serializers.py      | 50 +++++++++++++++++--
 .../apps/api/tests/test_serializers.py        |  4 +-
 .../api/v1/tests/test_views/test_courses.py   | 12 ++---
 .../apps/course_metadata/admin.py             |  7 +++
 .../migrations/0308_auto_20230111_1300.py     | 50 +++++++++++++++++++
 .../apps/course_metadata/models.py            | 14 ++++++
 .../apps/course_metadata/tests/factories.py   | 10 ++++
 7 files changed, 137 insertions(+), 10 deletions(-)
 create mode 100644 course_discovery/apps/course_metadata/migrations/0308_auto_20230111_1300.py

diff --git a/course_discovery/apps/api/serializers.py b/course_discovery/apps/api/serializers.py
index 25d9e6397e3..56841335ca1 100644
--- a/course_discovery/apps/api/serializers.py
+++ b/course_discovery/apps/api/serializers.py
@@ -41,8 +41,8 @@
     CourseType, Curriculum, CurriculumCourseMembership, CurriculumProgramMembership, Degree, DegreeAdditionalMetadata,
     DegreeCost, DegreeDeadline, Endorsement, Fact, GeoLocation, IconTextPairing, Image, LevelType, Mode, Organization,
     Pathway, Person, PersonAreaOfExpertise, PersonSocialNetwork, Position, Prerequisite, ProductMeta, ProductValue,
-    Program, ProgramLocationRestriction, ProgramType, Ranking, Seat, SeatType, Specialization, Subject, TaxiForm, Topic,
-    Track, Video
+    Program, ProgramLocationRestriction, ProgramType, Ranking, Seat, SeatType, Source, Specialization, Subject,
+    TaxiForm, Topic, Track, Video
 )
 from course_discovery.apps.course_metadata.utils import get_course_run_estimated_hours, parse_course_key_fragment
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
@@ -225,6 +225,13 @@ class Meta(TitleDescriptionSerializer.Meta):
         model = AdditionalPromoArea
 
 
+class SourceSerializer(BaseModelSerializer):
+    """Serializer for Source"""
+    class Meta:
+        model = Source
+        fields = ('name', 'slug', 'description')
+
+
 class FactSerializer(HeadingBlurbSerializer):
     """Serializer for Facts """
     class Meta(HeadingBlurbSerializer.Meta):
@@ -1268,6 +1275,7 @@ class CourseSerializer(TaggitSerializer, MinimalCourseSerializer):
     geolocation = GeoLocationSerializer(required=False, allow_null=True)
     location_restriction = CourseLocationRestrictionSerializer(required=False)
     in_year_value = ProductValueSerializer(required=False)
+    product_source = SourceSerializer(required=False)
 
     def get_organization_logo_override_url(self, obj):
         logo_image_override = getattr(obj, 'organization_logo_override', None)
@@ -1288,6 +1296,7 @@ def prefetch_queryset(cls, partner, queryset=None, course_runs=None):  # lint-am
             'partner',
             'extra_description',
             'additional_metadata',
+            'product_source',
             '_official_version',
             'canonical_course_run',
             'type',
@@ -1323,7 +1332,8 @@ class Meta(MinimalCourseSerializer.Meta):
             'enrollment_count', 'recent_enrollment_count', 'topics', 'partner', 'key_for_reruns', 'url_slug',
             'url_slug_history', 'url_redirects', 'course_run_statuses', 'editors', 'collaborators', 'skill_names',
             'skills', 'organization_short_code_override', 'organization_logo_override_url',
-            'enterprise_subscription_inclusion', 'geolocation', 'location_restriction', 'in_year_value'
+            'enterprise_subscription_inclusion', 'geolocation', 'location_restriction', 'in_year_value',
+            'product_source',
         )
         extra_kwargs = {
             'partner': {'write_only': True}
@@ -1400,6 +1410,17 @@ def update_product_meta(self, instance, product_meta_data):
             instance.product_meta.keywords.set(keywords, clear=True)
             instance.product_meta.save()
 
+    def update_product_source(self, instance, product_source):
+
+        if instance.product_source:
+            Source.objects.filter(id=instance.product_source.id).update(
+                **product_source)
+        else:
+            instance.product_source = Source.objects.create(
+                **product_source,
+            )
+            instance.save()
+
     def update_additional_metadata(self, instance, additional_metadata):
 
         facts = additional_metadata.pop('facts', None)
@@ -1456,6 +1477,9 @@ def update(self, instance, validated_data):
                 self.update_location_restriction(instance, location_restriction_data)
         if 'in_year_value' in validated_data:
             self.update_in_year_value(instance, validated_data.pop('in_year_value'))
+        if 'product_source' in validated_data:
+            self.update_product_source(
+                instance, validated_data.pop('product_source'))
         return super().update(instance, validated_data)
 
 
@@ -2048,6 +2072,7 @@ class ProgramSerializer(MinimalProgramSerializer):
     in_year_value = ProductValueSerializer(required=False)
     skill_names = serializers.SerializerMethodField()
     skills = serializers.SerializerMethodField()
+    product_source = SourceSerializer(required=False)
 
     @classmethod
     def prefetch_queryset(cls, partner, queryset=None):
@@ -2066,6 +2091,7 @@ def prefetch_queryset(cls, partner, queryset=None):
             'partner',
             'geolocation',
             'in_year_value',
+            'product_source',
             'location_restriction'
         ).prefetch_related(
             'excluded_course_runs',
@@ -2109,9 +2135,27 @@ class Meta(MinimalProgramSerializer.Meta):
             'staff', 'credit_redemption_overview', 'applicable_seat_types', 'instructor_ordering',
             'enrollment_count', 'topics', 'credit_value', 'enterprise_subscription_inclusion', 'geolocation',
             'location_restriction', 'is_2u_degree_program', 'in_year_value', 'skill_names', 'skills',
+            'product_source',
         )
         read_only_fields = ('enterprise_subscription_inclusion',)
 
+    def update_product_source(self, instance, product_source):
+
+        if instance.product_source:
+            Source.objects.filter(
+                id=instance.product_source.id).update(**product_source)
+        else:
+            instance.product_source = Source.objects.create(
+                **product_source,
+            )
+            instance.save()
+
+    def update(self, instance, validated_data):
+        if 'product_source' in validated_data:
+            self.update_product_source(
+                instance, validated_data.pop('product_source'))
+        return super().update(instance, validated_data)
+
 
 class PathwaySerializer(BaseModelSerializer):
     """ Serializer for Pathway. """
diff --git a/course_discovery/apps/api/tests/test_serializers.py b/course_discovery/apps/api/tests/test_serializers.py
index b282eac823c..273e850b2c7 100644
--- a/course_discovery/apps/api/tests/test_serializers.py
+++ b/course_discovery/apps/api/tests/test_serializers.py
@@ -30,7 +30,7 @@
     OrganizationSerializer, PathwaySerializer, PersonSerializer, PositionSerializer, PrerequisiteSerializer,
     ProductMetaSerializer, ProductValueSerializer, ProgramLocationRestrictionSerializer,
     ProgramsAffiliateWindowSerializer, ProgramSerializer, ProgramTypeAttrsSerializer, ProgramTypeSerializer,
-    RankingSerializer, SeatSerializer, SubjectSerializer, TaxiFormSerializer, TopicSerializer,
+    RankingSerializer, SeatSerializer, SourceSerializer, SubjectSerializer, TaxiFormSerializer, TopicSerializer,
     TypeaheadCourseRunSearchSerializer, TypeaheadProgramSearchSerializer, VideoSerializer,
     get_lms_course_url_for_archived, get_utm_source_for_user
 )
@@ -169,6 +169,7 @@ def get_expected_data(cls, course, course_skill, request):
             'full_description': course.full_description,
             'level_type': course.level_type.name_t,
             'extra_description': AdditionalPromoAreaSerializer(course.extra_description).data,
+            'product_source': SourceSerializer(course.product_source).data,
             'additional_metadata': AdditionalMetadataSerializer(course.additional_metadata).data,
             'subjects': [],
             'prerequisites': [],
@@ -1136,6 +1137,7 @@ def get_expected_data(cls, program, request, include_labels=True):
             'topics': [topic.name for topic in program.topics],
             'credit_value': program.credit_value,
             'enterprise_subscription_inclusion': program.enterprise_subscription_inclusion,
+            'product_source': SourceSerializer(program.product_source).data,
             'organization_short_code_override': program.organization_short_code_override,
             'organization_logo_override_url': program.organization_logo_override_url,
             'primary_subject_override': SubjectSerializer(program.primary_subject_override).data,
diff --git a/course_discovery/apps/api/v1/tests/test_views/test_courses.py b/course_discovery/apps/api/v1/tests/test_views/test_courses.py
index db125665d00..23168ff47bb 100644
--- a/course_discovery/apps/api/v1/tests/test_views/test_courses.py
+++ b/course_discovery/apps/api/v1/tests/test_views/test_courses.py
@@ -79,7 +79,7 @@ def test_get_uuid(self):
         """ Verify the endpoint returns the details for a single course with UUID. """
         url = reverse('api:v1:course-detail', kwargs={'key': self.course.uuid})
 
-        with self.assertNumQueries(44):
+        with self.assertNumQueries(47):
             response = self.client.get(url)
         assert response.status_code == 200
         assert response.data == self.serialize_course(self.course)
@@ -88,7 +88,7 @@ def test_get_exclude_deleted_programs(self):
         """ Verify the endpoint returns no deleted associated programs """
         ProgramFactory(courses=[self.course], status=ProgramStatus.Deleted)
         url = reverse('api:v1:course-detail', kwargs={'key': self.course.key})
-        with self.assertNumQueries(44):
+        with self.assertNumQueries(47):
             response = self.client.get(url)
         assert response.status_code == 200
         assert response.data.get('programs') == []
@@ -101,7 +101,7 @@ def test_get_include_deleted_programs(self):
         ProgramFactory(courses=[self.course], status=ProgramStatus.Deleted)
         url = reverse('api:v1:course-detail', kwargs={'key': self.course.key})
         url += '?include_deleted_programs=1'
-        with self.assertNumQueries(47):
+        with self.assertNumQueries(50):
             response = self.client.get(url)
         assert response.status_code == 200
         assert response.data == self.serialize_course(self.course, extra_context={'include_deleted_programs': True})
@@ -249,7 +249,7 @@ def test_list(self):
         """ Verify the endpoint returns a list of all courses. """
         url = reverse('api:v1:course-list')
 
-        with self.assertNumQueries(32):
+        with self.assertNumQueries(35, threshold=3):
             response = self.client.get(url)
         assert response.status_code == 200
         self.assertListEqual(
@@ -266,7 +266,7 @@ def test_list_query(self):
 
         # Known to be flaky prior to the addition of tearDown()
         # and logout() code which is the same number of additional queries
-        with self.assertNumQueries(62, threshold=3):
+        with self.assertNumQueries(65, threshold=3):
             response = self.client.get(url)
         self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
 
@@ -276,7 +276,7 @@ def test_list_key_filter(self):
         keys = ','.join([course.key for course in courses])
         url = '{root}?{params}'.format(root=reverse('api:v1:course-list'), params=urlencode({'keys': keys}))
 
-        with self.assertNumQueries(62, threshold=3):
+        with self.assertNumQueries(65, threshold=3):
             response = self.client.get(url)
         self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
 
diff --git a/course_discovery/apps/course_metadata/admin.py b/course_discovery/apps/course_metadata/admin.py
index 140abd75dbc..47bf13cdae9 100644
--- a/course_discovery/apps/course_metadata/admin.py
+++ b/course_discovery/apps/course_metadata/admin.py
@@ -90,6 +90,11 @@ class PositionInline(admin.TabularInline):
     extra = 0
 
 
+class SourceInline(admin.TabularInline):
+    model = Source
+    extra = 0
+
+
 class PersonSocialNetworkInline(admin.TabularInline):
     model = PersonSocialNetwork
     extra = 0
@@ -120,6 +125,7 @@ class ProductValueAdmin(admin.ModelAdmin):
 @admin.register(Course)
 class CourseAdmin(DjangoObjectActions, admin.ModelAdmin):
     form = CourseAdminForm
+    inline = (SourceInline,)
     list_display = ('uuid', 'key', 'key_for_reruns', 'title', 'draft',)
     list_filter = ('partner',)
     ordering = ('key', 'title',)
@@ -343,6 +349,7 @@ class ProgramLocationRestrictionAdmin(admin.ModelAdmin):
 @admin.register(Program)
 class ProgramAdmin(DjangoObjectActions, admin.ModelAdmin):
     form = ProgramAdminForm
+    inline = (SourceInline,)
     list_display = ('id', 'uuid', 'title', 'type', 'partner', 'status', 'hidden')
     list_filter = ('partner', 'type', 'status', ProgramEligibilityFilter, 'hidden')
     ordering = ('uuid', 'title', 'status')
diff --git a/course_discovery/apps/course_metadata/migrations/0308_auto_20230111_1300.py b/course_discovery/apps/course_metadata/migrations/0308_auto_20230111_1300.py
new file mode 100644
index 00000000000..e41746d051a
--- /dev/null
+++ b/course_discovery/apps/course_metadata/migrations/0308_auto_20230111_1300.py
@@ -0,0 +1,50 @@
+# Generated by Django 3.2.16 on 2023-01-11 13:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('course_metadata', '0307_additional_metadata_end_date'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('name', models.CharField(help_text='Name of the external source.', max_length=255)),
+                ('slug', django_extensions.db.fields.AutoSlugField(blank=True, editable=False, help_text='Leave this field blank to have the value generated automatically.', populate_from='name')),
+                ('description', models.CharField(blank=True, help_text='Description of the external source.', max_length=255)),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='product_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='courses', to='course_metadata.source'),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='product_source',
+            field=models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='course_metadata.source'),
+        ),
+        migrations.AddField(
+            model_name='historicalprogram',
+            name='product_source',
+            field=models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='course_metadata.source'),
+        ),
+        migrations.AddField(
+            model_name='program',
+            name='product_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='programs', to='course_metadata.source'),
+        ),
+    ]
diff --git a/course_discovery/apps/course_metadata/models.py b/course_discovery/apps/course_metadata/models.py
index 2120fb9ddc5..3fc257bad49 100644
--- a/course_discovery/apps/course_metadata/models.py
+++ b/course_discovery/apps/course_metadata/models.py
@@ -96,6 +96,18 @@ class Meta:
         abstract = True
 
 
+class Source(TimeStampedModel):
+    """
+    Source Model to find where a course or program originated from.
+    """
+    name = models.CharField(max_length=255, help_text=_('Name of the external source.'))
+    slug = AutoSlugField(
+        populate_from='name', editable=True, slugify_function=uslugify, overwrite_on_add=False,
+        help_text=_('Leave this field blank to have the value generated automatically.')
+    )
+    description = models.CharField(max_length=255, blank=True, help_text=_('Description of the external source.'))
+
+
 class CachedMixin:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1082,6 +1094,7 @@ class Course(DraftModelMixin, PkSearchableMixin, CachedMixin, TimeStampedModel):
     enrollment_count = models.IntegerField(
         null=True, blank=True, default=0, help_text=_('Total number of learners who have enrolled in this course')
     )
+    product_source = models.ForeignKey(Source, models.SET_NULL, null=True, blank=True, related_name='courses')
     recent_enrollment_count = models.IntegerField(
         null=True, blank=True, default=0, help_text=_(
             'Total number of learners who have enrolled in this course in the last 6 months'
@@ -2510,6 +2523,7 @@ class Program(PkSearchableMixin, TimeStampedModel):
     # NOTE (CCB): Editors of this field should validate the values to ensure only CourseRuns associated
     # with related Courses are stored.
     excluded_course_runs = models.ManyToManyField(CourseRun, blank=True)
+    product_source = models.ForeignKey(Source, models.SET_NULL, null=True, blank=True, related_name='programs')
     partner = models.ForeignKey(Partner, models.CASCADE, null=True, blank=False)
     overview = models.TextField(null=True, blank=True)
     total_hours_of_effort = models.PositiveSmallIntegerField(
diff --git a/course_discovery/apps/course_metadata/tests/factories.py b/course_discovery/apps/course_metadata/tests/factories.py
index 713305026f5..eb241416399 100644
--- a/course_discovery/apps/course_metadata/tests/factories.py
+++ b/course_discovery/apps/course_metadata/tests/factories.py
@@ -32,6 +32,14 @@ class AbstractHeadingBlurbModelFactory(factory.django.DjangoModelFactory):
     blurb = FuzzyText()
 
 
+class SourceFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Source
+
+    name = FuzzyText()
+    description = FuzzyText()
+
+
 class ImageFactory(AbstractMediaModelFactory):
     height = 100
     width = 100
@@ -359,6 +367,7 @@ class CourseFactory(SalesforceRecordFactory):
     organization_short_code_override = FuzzyText()
     canonical_course_run = None
     extra_description = factory.SubFactory(AdditionalPromoAreaFactory)
+    product_source = factory.SubFactory(SourceFactory)
     additional_metadata = factory.SubFactory(AdditionalMetadataFactory)
     additional_information = FuzzyText()
     faq = FuzzyText()
@@ -644,6 +653,7 @@ class Meta:
     organization_short_code_override = FuzzyText()
     organization_logo_override = FuzzyText(suffix=".png")
     primary_subject_override = factory.SubFactory(SubjectFactory)
+    product_source = factory.SubFactory(SourceFactory)
     level_type_override = factory.SubFactory(LevelTypeFactory)
     language_override = factory.Iterator(LanguageTag.objects.all())
     taxi_form = factory.SubFactory(TaxiFormFactory)

From b87bf6e19296ed1b9798c65ad6587da978b18c82 Mon Sep 17 00:00:00 2001
From: "afaq.shuaib" <afaq.shuaib09@arbisoft.com>
Date: Wed, 28 Dec 2022 00:53:48 +0500
Subject: [PATCH 03/14] feat: add googleapiclent to communicate with drive
 images

---
 .../apps/course_metadata/constants.py         |   5 +
 .../apps/course_metadata/googleapi_client.py  |  51 +++++++
 .../course_metadata/tests/test_googleapi.py   |  61 ++++++++
 .../apps/course_metadata/tests/test_utils.py  | 132 +++++++++++++++++-
 .../apps/course_metadata/utils.py             | 132 +++++++++++-------
 requirements/base.in                          |   3 +
 requirements/local.txt                        |   8 +-
 requirements/production.txt                   |   6 +
 8 files changed, 349 insertions(+), 49 deletions(-)
 create mode 100644 course_discovery/apps/course_metadata/googleapi_client.py
 create mode 100644 course_discovery/apps/course_metadata/tests/test_googleapi.py

diff --git a/course_discovery/apps/course_metadata/constants.py b/course_discovery/apps/course_metadata/constants.py
index 308ad59668b..b88f2503bd8 100644
--- a/course_discovery/apps/course_metadata/constants.py
+++ b/course_discovery/apps/course_metadata/constants.py
@@ -15,6 +15,11 @@
     'image/svg+xml': 'svg'  # SVG image will be converted into PNG, not stored as SVG
 }
 
+DRIVE_LINK_PATTERNS = [r"https://docs\.google\.com/uc\?id=\w+",
+                       r"https://drive\.google\.com/file/d/\w+/view?usp=sharing"]
+
+GOOGLE_CLIENT_API_SCOPE = ['https://www.googleapis.com/auth/drive']
+
 
 class PathwayType(Enum):
     """ Allowed values for Pathway.pathway_type """
diff --git a/course_discovery/apps/course_metadata/googleapi_client.py b/course_discovery/apps/course_metadata/googleapi_client.py
new file mode 100644
index 00000000000..15ec973c1d1
--- /dev/null
+++ b/course_discovery/apps/course_metadata/googleapi_client.py
@@ -0,0 +1,51 @@
+import logging
+import re
+
+from django.conf import settings
+from googleapiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+
+from course_discovery.apps.course_metadata.constants import GOOGLE_CLIENT_API_SCOPE
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleAPIClient:
+    """
+    API Client for Google API to communicate with drive files
+    """
+
+    def __init__(self):
+        try:
+            credentials = ServiceAccountCredentials.from_json_keyfile_dict(
+                settings.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS, GOOGLE_CLIENT_API_SCOPE)
+            self.service = build('drive', 'v3', credentials=credentials)
+            logger.info('[Connection Successful]: Successful connection with google service account')
+        except Exception as ex:  # pylint: disable=broad-except
+            logger.exception(f'[Connection Failed]: Failed to connect with google service account error_message: {ex}')
+
+    def get_file_metadata(self, url):
+        try:
+            file_id = self.get_file_id_from_url(url)
+            file = self.service.files().get(fileId=file_id).execute()  # pylint: disable=no-member
+            logger.info(f'[File Found]: Found google file {file_id} on requesting {url}')
+            return file
+        except Exception as ex:  # pylint: disable=broad-except
+            logger.exception(f'[File Not Found]: No file found for id: {file_id} error_message: {ex}')
+        return None
+
+    @staticmethod
+    def get_file_id_from_url(url):
+        match = re.search(r'id=(\w+)', url) or re.search(r'/(?:file/d/|uc\?id=)([-\w]{25,})(?:[&/]|$)', url)
+        return match.group(1) if match else None
+
+    def download_file_by_url(self, url):
+        content = None
+        try:
+            file_id = self.get_file_id_from_url(url)
+            request = self.service.files().get_media(fileId=file_id)  # pylint: disable=no-member
+            content = request.execute()
+            logger.info(f'[File Downloaded]: Downloading google file {file_id}')
+        except Exception as ex:  # pylint: disable=broad-except
+            logger.exception(f'[File Not Downloaded]: No file found for id: {file_id} error_message: {ex}')
+        return content
diff --git a/course_discovery/apps/course_metadata/tests/test_googleapi.py b/course_discovery/apps/course_metadata/tests/test_googleapi.py
new file mode 100644
index 00000000000..0b438f45e55
--- /dev/null
+++ b/course_discovery/apps/course_metadata/tests/test_googleapi.py
@@ -0,0 +1,61 @@
+from unittest import mock
+
+import ddt
+from django.test import TestCase
+
+from course_discovery.apps.course_metadata.googleapi_client import GoogleAPIClient
+
+
+@ddt.ddt
+class GoogleAPIClientTests(TestCase):
+
+    Google_DRIVE_API_TEST_DATA = [
+        ('https://docs.google.com/uc?id=abc123Id', 'abc123Id'),
+        ('https://drive.google.com/file/d/1Xv36dVXFC-eU2Oks_EcRdbtgv47D-osP/view?usp=sharing', '1Xv36dVXFC-eU2Oks_EcRdbtgv47D-osP'),  # pylint: disable=line-too-long
+    ]
+
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
+    def test_connection_with_google(self, mock_googleapi_connection, mock_account_credentials, mock_logger):
+        GoogleAPIClient()
+        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_googleapi_connection.called is True
+        mock_logger.info.assert_called_with(
+            '[Connection Successful]: Successful connection with google service account'
+        )
+
+    @ddt.data(*Google_DRIVE_API_TEST_DATA)
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
+    @ddt.unpack
+    def test_get_file_id_from_url(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials):  # pylint: disable=line-too-long
+        client = GoogleAPIClient()
+        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_googleapi_connection.called is True
+        file_id = client.get_file_id_from_url(file_url)
+        assert file_id == expected_file_id
+
+    @ddt.data(*Google_DRIVE_API_TEST_DATA)
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
+    @ddt.unpack
+    def test_get_file_metadata(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials, mock_logger):  # pylint: disable=line-too-long
+        client = GoogleAPIClient()
+        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_googleapi_connection.called is True
+        client.get_file_metadata(file_url)
+        mock_logger.info.assert_called_with(
+            f'[File Found]: Found google file {expected_file_id} on requesting {file_url}'
+        )
+
+    @ddt.data(*Google_DRIVE_API_TEST_DATA)
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
+    @ddt.unpack
+    def test_download_file_by_url(self, file_url, expected_file_id, _mock_googleapi_connection, _mock_account_credentials, mock_logger):  # pylint: disable=line-too-long
+        client = GoogleAPIClient()
+        client.download_file_by_url(file_url)
+        mock_logger.info.assert_called_with(f'[File Downloaded]: Downloading google file {expected_file_id}')
diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index 603d57bc64b..f9f673f2c25 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -26,7 +26,8 @@
 from course_discovery.apps.course_metadata.tests.mixins import MarketingSiteAPIClientTestMixin
 from course_discovery.apps.course_metadata.utils import (
     calculated_seat_upgrade_deadline, clean_html, convert_svg_to_png_from_url, create_missing_entitlement,
-    ensure_draft_world, serialize_entitlement_for_ecommerce_api, serialize_seat_for_ecommerce_api, transform_skills_data
+    download_and_save_course_image, download_and_save_program_image, ensure_draft_world, is_google_drive_url,
+    serialize_entitlement_for_ecommerce_api, serialize_seat_for_ecommerce_api, transform_skills_data
 )
 
 
@@ -719,3 +720,132 @@ class TestConvertSvgToPngFromUrl(TestCase):
     def test_convert_svg_to_png_from_url(self, _svg2png_mock):
         """Verify that convert_svg_to_png_from_url will return a valid value"""
         assert convert_svg_to_png_from_url('https://www.svgimageurl.com') is not None
+
+
+@ddt.ddt
+class TestIsGoogleDriveUrl(TestCase):
+    """Test is google drive url"""
+    @ddt.data(
+        ('https://docs.google.com/uc?id=abcd123id', True),
+        ('https://example.com/image.jpg', False),
+    )
+    @ddt.unpack
+    def test_is_google_drive_url(self, url, expected):
+        """Verify that is_google_drive_url will return a valid value"""
+        assert is_google_drive_url(url) is expected
+
+
+@ddt.ddt
+class TestDownloadAndSaveImage(TestCase):
+    """ Test to download and save image """
+
+    IMG_CONTENT = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00' \
+                  b'\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00' \
+                  b'IEND\xaeB`\x82'
+
+    def mock_image_response(self, status=200, body=None, content_type='image/jpeg', url='https://example.com/image.jpg'):  # pylint: disable=invalid-name
+        """ Mock image response """
+        body = body or b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00' \
+                       b'\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00' \
+                       b'IEND\xaeB`\x82'
+        image_url = url
+        responses.add(
+            responses.GET,
+            image_url,
+            body=body,
+            status=status,
+            content_type=content_type
+        )
+        return image_url, body
+
+    @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
+    def test_download_and_save_course_image_using_drive_link(self, mock_get_file_from_drive_link):
+        """ Verify that download_and_save_course_image will save image in course model using drive link """
+        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
+        mock_get_file_from_drive_link = mock.Mock()
+        mock_get_file_from_drive_link.return_value = (self.IMG_CONTENT, 'image/jpeg')
+        assert course.card_image_url == image_url
+        assert course.image is not None
+
+    @responses.activate
+    def test_download_and_save_course_image_using_request_library(self):
+        """ Verify that download_and_save_course_image will save image in course model using request response """
+        image_url = 'https://example.com/image.jpg'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        image_url, content = self.mock_image_response()
+        response = requests.get('https://example.com/image.jpg')
+        assert response.content == content
+        assert course.card_image_url == image_url
+        assert course.image is not None
+
+    @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
+    @responses.activate
+    def test_download_and_save_course_image_with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
+        """ Verify that download_and_save_course_image will not save image in course model """
+        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
+        mock_get_file_from_drive_link = mock.Mock()
+        mock_get_file_from_drive_link.return_value = (b'invalid', 'text/plain')
+        assert course.card_image_url == image_url
+        assert not bool(course.image)
+
+    @responses.activate
+    def test_download_and_save_course_image_with_invalid_content_type_using_request_library(self):
+        """ Verify that download_and_save_course_image will not save image in course model """
+        image_url = 'https://www.example.com/image.pdf'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
+        response = requests.get('https://www.example.com/image.pdf')
+        assert response.content == content
+        assert not bool(course.image)
+
+    @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
+    def test_download_and_save_program_image_using_drive_link(self, mock_get_file_from_drive_link):
+        """ Verify that download_and_save_program_image will save image in program model """
+        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        program = ProgramFactory(card_image_url=image_url, card_image=None)
+        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
+        mock_get_file_from_drive_link.return_value = (self.IMG_CONTENT, 'image/jpeg')
+        assert program.card_image_url == image_url
+        assert program.card_image is not None
+
+    @responses.activate
+    def test_download_and_save_program_image_using_request_library(self):
+        """ Verify that download_and_save_program_image will save image in program model using request response """
+        image_url = 'https://www.example.com'
+        program = ProgramFactory(card_image_url=image_url, card_image=None)
+        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        image_url, content = self.mock_image_response()
+        response = requests.get('https://example.com/image.jpg')
+        assert response.content == content
+        assert program.card_image is not None
+
+    @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
+    def test_download_and_save_program_image_with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
+        """ Verify that download_and_save_program_image will not save image in program model using drive link """
+        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        program = ProgramFactory(card_image_url=image_url, card_image=None)
+        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
+        mock_get_file_from_drive_link.return_value = (b'invalid', 'text/plain')
+        assert program.card_image_url == image_url
+        assert not bool(program.card_image)
+
+    @responses.activate
+    def test_download_and_save_program_image_with_invalid_content_type_using_request_library(self):
+        """ Verify that download_and_save_program_image will not save image in program model using request response """
+        image_url = 'https://www.example.com/image.pdf'
+        program = ProgramFactory(card_image_url=image_url, card_image=None)
+        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
+        response = requests.get('https://www.example.com/image.pdf')
+        assert response.content == content
+        assert not bool(program.card_image)
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index ab4b7bb3bc7..bdd25769822 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -1,6 +1,7 @@
 import datetime
 import logging
 import random
+import re
 import string
 import uuid
 from tempfile import NamedTemporaryFile
@@ -21,10 +22,11 @@
 
 from course_discovery.apps.core.models import SalesforceConfiguration
 from course_discovery.apps.core.utils import serialize_datetime
-from course_discovery.apps.course_metadata.constants import IMAGE_TYPES
+from course_discovery.apps.course_metadata.constants import DRIVE_LINK_PATTERNS, IMAGE_TYPES
 from course_discovery.apps.course_metadata.exceptions import (
     EcommerceSiteAPIClientException, MarketingSiteAPIClientException
 )
+from course_discovery.apps.course_metadata.googleapi_client import GoogleAPIClient
 from course_discovery.apps.course_metadata.salesforce import SalesforceUtil
 from course_discovery.apps.publisher.utils import VALID_CHARS_IN_COURSE_NUM_AND_ORG_KEY
 
@@ -690,6 +692,22 @@ def clean_html(content):
     return cleaned
 
 
+def get_file_from_drive_link(image_url):
+    """
+    Helper method to get the file content_type and content from a drive link
+
+    Keyword Arguments:
+        image_url {str} -- drive link of the file
+
+    Returns:
+        tuple --  content_type and content of the file
+    """
+    google_api_client = GoogleAPIClient()
+    content_type = google_api_client.get_file_metadata(image_url).get('mimeType')
+    content = google_api_client.download_file_by_url(image_url)
+    return content_type, content
+
+
 def download_and_save_course_image(course, image_url, data_field='image', headers=None):
     """
     Helper method to download an image from a provided image url and save it
@@ -697,38 +715,45 @@ def download_and_save_course_image(course, image_url, data_field='image', header
     """
     try:
         image_url = get_downloadable_url_from_drive_link(image_url)
-        response = requests.get(image_url, headers=headers)  # pylint: disable=missing-timeout
 
-        if response.status_code == requests.codes.ok:  # pylint: disable=no-member
-            content_type = response.headers['Content-Type'].lower()
+        if is_google_drive_url(image_url):
+            content_type, content = get_file_from_drive_link(image_url)
             extension = IMAGE_TYPES.get(content_type)
+        else:
+            response = requests.get(image_url, headers=headers)  # pylint: disable=missing-timeout
 
-            if extension:
-                filename = '{uuid}.{extension}'.format(uuid=str(course.uuid), extension=extension)
-                # TODO: Get field from _meta.get_field. Tried that approach initially but was getting
-                # field save errors for some reasons.
-                if data_field == 'image':
-                    course.image.save(filename, ContentFile(response.content))
-                elif data_field == 'organization_logo_override':
-                    image_file = ContentFile(response.content)
-                    if extension == 'svg':
-                        filename = '{uuid}.png'.format(uuid=str(course.uuid))
-                        image_file = convert_svg_to_png_from_url(image_url)
-                    if image_file:
-                        course.organization_logo_override.save(filename, image_file)
-                    else:
-                        logger.error('Update organization logo override failed for course [%s]', course.key)
-                        return False
-                logger.info('Image for course [%s] successfully updated.', course.key)
-                return True
-            else:
-                # pylint: disable=line-too-long
-                msg = 'Image retrieved for course [%s] from [%s] has an unknown content type [%s] and will not be saved.'
-                logger.error(msg, course.key, image_url, content_type)
+            if response.status_code == requests.codes.ok:  # pylint: disable=no-member
+                content_type = response.headers['Content-Type'].lower()
+                extension = IMAGE_TYPES.get(content_type)
+                content = response.content
 
+            else:
+                msg = 'Failed to download image for course [%s] from [%s]! Response was [%d]:\n%s'
+                logger.error(msg, course.key, image_url, response.status_code, response.content)
+                return False
+
+        if extension:
+            filename = '{uuid}.{extension}'.format(uuid=str(course.uuid), extension=extension)
+            # TODO: Get field from _meta.get_field. Tried that approach initially but was getting
+            # field save errors for some reasons.
+            if data_field == 'image':
+                course.image.save(filename, ContentFile(content))
+            elif data_field == 'organization_logo_override':
+                image_file = ContentFile(response.content)
+                if extension == 'svg':
+                    filename = '{uuid}.png'.format(uuid=str(course.uuid))
+                    image_file = convert_svg_to_png_from_url(image_url)
+                if image_file:
+                    course.organization_logo_override.save(filename, image_file)
+                else:
+                    logger.error('Update organization logo override failed for course [%s]', course.key)
+                    return False
+            logger.info('Image for course [%s] successfully updated.', course.key)
+            return True
         else:
-            msg = 'Failed to download image for course [%s] from [%s]! Response was [%d]:\n%s'
-            logger.error(msg, course.key, image_url, response.status_code, response.content)
+            msg = 'Image retrieved for course [%s] from [%s] has an unknown content type [%s] and will not be saved.'
+            logger.error(msg, course.key, image_url, content_type)
+
     except Exception:  # pylint: disable=broad-except
         logger.exception('An unknown exception occurred while downloading image for course [%s]', course.key)
     return False
@@ -761,6 +786,13 @@ def get_downloadable_url_from_drive_link(file_path):
     return file_path
 
 
+def is_google_drive_url(url):
+    """
+    Helper method to check if the file url is a drive url or not
+    """
+    return any(re.match(pattern, url) for pattern in DRIVE_LINK_PATTERNS)
+
+
 def download_and_save_program_image(program, image_url, data_field='image', headers=None):
     """
     Helper method to download an image from a provided image url and save it
@@ -769,30 +801,36 @@ def download_and_save_program_image(program, image_url, data_field='image', head
     # TODO: refactor and merge program image download to use the same code as course image download
     try:
         image_url = get_downloadable_url_from_drive_link(image_url)
-        response = requests.get(image_url, headers=headers)  # pylint: disable=missing-timeout
 
-        if response.status_code == requests.codes.ok:  # pylint: disable=no-member
-            content_type = response.headers['Content-Type'].lower()
+        if is_google_drive_url(image_url):
+            content_type, content = get_file_from_drive_link(image_url)
             extension = IMAGE_TYPES.get(content_type)
+        else:
+            response = requests.get(image_url, headers=headers)  # pylint: disable=missing-timeout
 
-            if extension:
-                filename = '{uuid}.{extension}'.format(uuid=str(program.uuid), extension=extension)
-                # TODO: Get field from _meta.get_field. Tried that approach initially but was getting
-                # field save errors for some reasons.
-                if data_field == 'image':
-                    program.card_image.save(filename, ContentFile(response.content))
-                elif data_field == 'organization_logo_override':
-                    program.organization_logo_override.save(filename, ContentFile(response.content))
-                logger.info('Image for program [%s] successfully updated.', program.title)
-                return True
+            if response.status_code == requests.codes.ok:  # pylint: disable=no-member
+                content_type = response.headers['Content-Type'].lower()
+                extension = IMAGE_TYPES.get(content_type)
+                content = response.content
             else:
-                # pylint: disable=line-too-long
-                msg = 'Image retrieved for program [%s] from [%s] has an unknown content type [%s] and will not be saved.'
-                logger.error(msg, program.title, image_url, content_type)
-
+                msg = 'Failed to download image for program [%s] from [%s]! Response was [%d]:\n%s'
+                logger.error(msg, program.title, image_url, response.status_code, response.content)
+                return False
+
+        if extension:
+            filename = '{uuid}.{extension}'.format(uuid=str(program.uuid), extension=extension)
+            # TODO: Get field from _meta.get_field. Tried that approach initially but was getting
+            # field save errors for some reasons.
+            if data_field == 'image':
+                program.card_image.save(filename, ContentFile(content))
+            elif data_field == 'organization_logo_override':
+                program.organization_logo_override.save(filename, ContentFile(content))
+            logger.info('Image for program [%s] successfully updated.', program.title)
+            return True
         else:
-            msg = 'Failed to download image for program [%s] from [%s]! Response was [%d]:\n%s'
-            logger.error(msg, program.title, image_url, response.status_code, response.content)
+            msg = 'Image retrieved for program [%s] from [%s] has an unknown content type [%s] and will not be saved.'
+            logger.error(msg, program.title, image_url, content_type)
+
     except Exception:  # pylint: disable=broad-except
         logger.exception('An unknown exception occurred while downloading image for program [%s]', program.title)
     return False
diff --git a/requirements/base.in b/requirements/base.in
index fcf803c4b0e..0544690b072 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -53,9 +53,12 @@ edx-opaque-keys
 edx-rest-api-client
 elasticsearch
 elasticsearch-dsl
+google-api-python-client
+google-auth-httplib2
 gspread
 html2text
 lxml
+oauth2client
 jsonfield
 markdown
 pillow
diff --git a/requirements/local.txt b/requirements/local.txt
index 400f862b331..ab91a98a854 100644
--- a/requirements/local.txt
+++ b/requirements/local.txt
@@ -432,7 +432,10 @@ google-auth==2.16.0
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==0.8.0
-    # via gspread
+google-api-python-client==2.70.0
+    # via -r requirements/base.in
+google-auth-httplib2==0.1.0
+    # via google-api-python-client
 gspread==5.7.2
     # via -r requirements/base.in
 h11==0.14.0
@@ -510,6 +513,9 @@ openedx-events==4.1.0
     # via
     #   edx-event-bus-kafka
     #   taxonomy-connector
+
+oauth2client==4.1.3
+    # via -r requirements/base.in
 oscrypto==1.3.0
     # via snowflake-connector-python
 outcome==1.2.0
diff --git a/requirements/production.txt b/requirements/production.txt
index a0d9ba85e17..70f8bf0f932 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -344,6 +344,10 @@ google-auth==2.16.0
     #   gspread
 google-auth-oauthlib==0.8.0
     # via gspread
+google-api-python-client==2.70.0
+    # via -r requirements/base.in
+google-auth-httplib2==0.1.0
+    # via google-api-python-client
 greenlet==2.0.1
     # via gevent
 gspread==5.7.2
@@ -404,6 +408,8 @@ openedx-events==4.1.0
     # via
     #   edx-event-bus-kafka
     #   taxonomy-connector
+oauth2client==4.1.3
+    # via -r requirements/base.in
 oscrypto==1.3.0
     # via snowflake-connector-python
 packaging==23.0

From c59a760cf0cbfd58c2f02a3d2f05d1e2d657b931 Mon Sep 17 00:00:00 2001
From: "afaq.shuaib" <afaq.shuaib09@arbisoft.com>
Date: Mon, 2 Jan 2023 23:15:45 +0500
Subject: [PATCH 04/14] fix: line-too long quality issue

---
 course_discovery/apps/course_metadata/tests/test_utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index f9f673f2c25..071b4b7190d 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -743,8 +743,7 @@ class TestDownloadAndSaveImage(TestCase):
                   b'\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00' \
                   b'IEND\xaeB`\x82'
 
-    def mock_image_response(self, status=200, body=None, content_type='image/jpeg', url='https://example.com/image.jpg'):  # pylint: disable=invalid-name
-        """ Mock image response """
+    def mock_image_response(self, status=200, body=None, content_type='image/jpeg', url='https://example.com/image.jpg'):  # pylint: disable=line-too-long
         body = body or b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00' \
                        b'\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00' \
                        b'IEND\xaeB`\x82'

From 46425ad432957aa25e78a51e5545d2cabddfe392 Mon Sep 17 00:00:00 2001
From: "afaq.shuaib" <afaq.shuaib09@arbisoft.com>
Date: Tue, 3 Jan 2023 00:05:08 +0500
Subject: [PATCH 05/14] fix: missing timeout for request

---
 course_discovery/apps/course_metadata/tests/test_utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index 071b4b7190d..bc54731f1ab 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -776,7 +776,7 @@ def test_download_and_save_course_image_using_request_library(self):
         course = CourseFactory(card_image_url=image_url, image=None)
         download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
         image_url, content = self.mock_image_response()
-        response = requests.get('https://example.com/image.jpg')
+        response = requests.get('https://example.com/image.jpg', timeout=5)
         assert response.content == content
         assert course.card_image_url == image_url
         assert course.image is not None
@@ -801,7 +801,7 @@ def test_download_and_save_course_image_with_invalid_content_type_using_request_
         course = CourseFactory(card_image_url=image_url, image=None)
         download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
         image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
-        response = requests.get('https://www.example.com/image.pdf')
+        response = requests.get('https://www.example.com/image.pdf', timeout=5)
         assert response.content == content
         assert not bool(course.image)
 
@@ -823,7 +823,7 @@ def test_download_and_save_program_image_using_request_library(self):
         program = ProgramFactory(card_image_url=image_url, card_image=None)
         download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
         image_url, content = self.mock_image_response()
-        response = requests.get('https://example.com/image.jpg')
+        response = requests.get('https://example.com/image.jpg', timeout=5)
         assert response.content == content
         assert program.card_image is not None
 
@@ -845,6 +845,6 @@ def test_download_and_save_program_image_with_invalid_content_type_using_request
         program = ProgramFactory(card_image_url=image_url, card_image=None)
         download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
         image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
-        response = requests.get('https://www.example.com/image.pdf')
+        response = requests.get('https://www.example.com/image.pdf', timeout=5)
         assert response.content == content
         assert not bool(program.card_image)

From a2165acc30a59052aedb5377c66753dcbc843c9d Mon Sep 17 00:00:00 2001
From: aliadnan <ali.adnan@arbisoft.com>
Date: Wed, 11 Jan 2023 19:13:18 +0500
Subject: [PATCH 06/14] feat: Update google client to handle impersonating
 email to access google drive images

---
 .../apps/course_metadata/constants.py         |  2 +-
 .../apps/course_metadata/googleapi_client.py  |  8 +--
 .../course_metadata/tests/test_googleapi.py   | 26 ++++++----
 requirements/base.in                          |  1 -
 requirements/docs.txt                         |  2 +-
 requirements/local.txt                        | 50 +++++++++++++------
 requirements/pip_tools.txt                    | 10 ++--
 requirements/production.txt                   | 42 ++++++++++++----
 8 files changed, 92 insertions(+), 49 deletions(-)

diff --git a/course_discovery/apps/course_metadata/constants.py b/course_discovery/apps/course_metadata/constants.py
index b88f2503bd8..3dc91c41b33 100644
--- a/course_discovery/apps/course_metadata/constants.py
+++ b/course_discovery/apps/course_metadata/constants.py
@@ -18,7 +18,7 @@
 DRIVE_LINK_PATTERNS = [r"https://docs\.google\.com/uc\?id=\w+",
                        r"https://drive\.google\.com/file/d/\w+/view?usp=sharing"]
 
-GOOGLE_CLIENT_API_SCOPE = ['https://www.googleapis.com/auth/drive']
+GOOGLE_CLIENT_API_SCOPE = ['https://www.googleapis.com/auth/drive.readonly']
 
 
 class PathwayType(Enum):
diff --git a/course_discovery/apps/course_metadata/googleapi_client.py b/course_discovery/apps/course_metadata/googleapi_client.py
index 15ec973c1d1..f5e1848bbcb 100644
--- a/course_discovery/apps/course_metadata/googleapi_client.py
+++ b/course_discovery/apps/course_metadata/googleapi_client.py
@@ -2,8 +2,8 @@
 import re
 
 from django.conf import settings
+from google.oauth2.service_account import Credentials
 from googleapiclient.discovery import build
-from oauth2client.service_account import ServiceAccountCredentials
 
 from course_discovery.apps.course_metadata.constants import GOOGLE_CLIENT_API_SCOPE
 
@@ -17,8 +17,10 @@ class GoogleAPIClient:
 
     def __init__(self):
         try:
-            credentials = ServiceAccountCredentials.from_json_keyfile_dict(
-                settings.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS, GOOGLE_CLIENT_API_SCOPE)
+            credentials = Credentials.from_service_account_info(
+                settings.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS, scopes=GOOGLE_CLIENT_API_SCOPE
+            )
+            credentials = credentials.with_subject(settings.LOADER_INGESTION_CONTACT_EMAIL)
             self.service = build('drive', 'v3', credentials=credentials)
             logger.info('[Connection Successful]: Successful connection with google service account')
         except Exception as ex:  # pylint: disable=broad-except
diff --git a/course_discovery/apps/course_metadata/tests/test_googleapi.py b/course_discovery/apps/course_metadata/tests/test_googleapi.py
index 0b438f45e55..78ad7fb8612 100644
--- a/course_discovery/apps/course_metadata/tests/test_googleapi.py
+++ b/course_discovery/apps/course_metadata/tests/test_googleapi.py
@@ -15,35 +15,38 @@ class GoogleAPIClientTests(TestCase):
     ]
 
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
-    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials.with_subject')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials')
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
-    def test_connection_with_google(self, mock_googleapi_connection, mock_account_credentials, mock_logger):
+    def test_connection_with_google(self, mock_googleapi_connection, mock_account_credentials, _mock_with_subject, mock_logger):  # pylint: disable=line-too-long
         GoogleAPIClient()
-        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_account_credentials.from_service_account_info.called is True
         assert mock_googleapi_connection.called is True
         mock_logger.info.assert_called_with(
             '[Connection Successful]: Successful connection with google service account'
         )
 
     @ddt.data(*Google_DRIVE_API_TEST_DATA)
-    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials.with_subject')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials')
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
     @ddt.unpack
-    def test_get_file_id_from_url(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials):  # pylint: disable=line-too-long
+    def test_get_file_id_from_url(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials, _mock_with_subject):  # pylint: disable=line-too-long
         client = GoogleAPIClient()
-        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_account_credentials.from_service_account_info.called is True
         assert mock_googleapi_connection.called is True
         file_id = client.get_file_id_from_url(file_url)
         assert file_id == expected_file_id
 
     @ddt.data(*Google_DRIVE_API_TEST_DATA)
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
-    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials.with_subject')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials')
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
     @ddt.unpack
-    def test_get_file_metadata(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials, mock_logger):  # pylint: disable=line-too-long
+    def test_get_file_metadata(self, file_url, expected_file_id, mock_googleapi_connection, mock_account_credentials, _mock_with_subject, mock_logger):  # pylint: disable=line-too-long
         client = GoogleAPIClient()
-        assert mock_account_credentials.from_json_keyfile_dict.called is True
+        assert mock_account_credentials.from_service_account_info.called is True
         assert mock_googleapi_connection.called is True
         client.get_file_metadata(file_url)
         mock_logger.info.assert_called_with(
@@ -52,10 +55,11 @@ def test_get_file_metadata(self, file_url, expected_file_id, mock_googleapi_conn
 
     @ddt.data(*Google_DRIVE_API_TEST_DATA)
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.logger')
-    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.ServiceAccountCredentials')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials.with_subject')
+    @mock.patch('course_discovery.apps.course_metadata.googleapi_client.Credentials')
     @mock.patch('course_discovery.apps.course_metadata.googleapi_client.build')
     @ddt.unpack
-    def test_download_file_by_url(self, file_url, expected_file_id, _mock_googleapi_connection, _mock_account_credentials, mock_logger):  # pylint: disable=line-too-long
+    def test_download_file_by_url(self, file_url, expected_file_id, _mock_googleapi_connection, _mock_account_credentials, _mock_with_subject, mock_logger):  # pylint: disable=line-too-long
         client = GoogleAPIClient()
         client.download_file_by_url(file_url)
         mock_logger.info.assert_called_with(f'[File Downloaded]: Downloading google file {expected_file_id}')
diff --git a/requirements/base.in b/requirements/base.in
index 0544690b072..b521af5d875 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -58,7 +58,6 @@ google-auth-httplib2
 gspread
 html2text
 lxml
-oauth2client
 jsonfield
 markdown
 pillow
diff --git a/requirements/docs.txt b/requirements/docs.txt
index 58fa849cc8e..9a4e5e8f35b 100644
--- a/requirements/docs.txt
+++ b/requirements/docs.txt
@@ -75,7 +75,7 @@ sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
-urllib3==1.26.13
+urllib3==1.26.14
     # via
     #   elasticsearch
     #   requests
diff --git a/requirements/local.txt b/requirements/local.txt
index ab91a98a854..6b56c60e102 100644
--- a/requirements/local.txt
+++ b/requirements/local.txt
@@ -64,9 +64,9 @@ boltons==21.0.0
     #   face
     #   glom
     #   semgrep
-boto3==1.26.46
+boto3==1.26.48
     # via django-ses
-botocore==1.29.46
+botocore==1.29.48
     # via
     #   boto3
     #   s3transfer
@@ -134,7 +134,7 @@ coreschema==0.0.4
     # via
     #   coreapi
     #   drf-yasg
-coverage[toml]==7.0.4
+coverage[toml]==7.0.5
     # via
     #   -r requirements/test.in
     #   pytest-cov
@@ -412,7 +412,7 @@ face==22.0.0
     # via glom
 factory-boy==3.2.1
     # via -r requirements/test.in
-faker==16.1.0
+faker==16.3.0
     # via factory-boy
 fastavro==1.7.0
     # via openedx-events
@@ -427,21 +427,35 @@ future==0.18.2
     # via pyjwkest
 glom==22.1.0
     # via semgrep
+google-api-core==2.11.0
+    # via google-api-python-client
+google-api-python-client==2.72.0
+    # via -r requirements/base.in
 google-auth==2.16.0
     # via
+    #   google-api-core
+    #   google-api-python-client
+    #   google-auth-httplib2
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==0.8.0
-google-api-python-client==2.70.0
-    # via -r requirements/base.in
 google-auth-httplib2==0.1.0
-    # via google-api-python-client
+    # via
+    #   -r requirements/base.in
+    #   google-api-python-client
+google-auth-oauthlib==0.8.0
+    # via gspread
+googleapis-common-protos==1.58.0
+    # via google-api-core
 gspread==5.7.2
     # via -r requirements/base.in
 h11==0.14.0
     # via wsproto
 html2text==2020.1.16
     # via -r requirements/base.in
+httplib2==0.21.0
+    # via
+    #   google-api-python-client
+    #   google-auth-httplib2
 idna==3.4
     # via
     #   requests
@@ -513,9 +527,6 @@ openedx-events==4.1.0
     # via
     #   edx-event-bus-kafka
     #   taxonomy-connector
-
-oauth2client==4.1.3
-    # via -r requirements/base.in
 oscrypto==1.3.0
     # via snowflake-connector-python
 outcome==1.2.0
@@ -535,7 +546,7 @@ paramiko==2.12.0
     # via docker
 path==16.6.0
     # via edx-i18n-tools
-pbr==5.11.0
+pbr==5.11.1
     # via stevedore
 peewee==3.15.4
     # via semgrep
@@ -557,6 +568,10 @@ polib==1.1.1
     # via edx-i18n-tools
 prompt-toolkit==3.0.36
     # via click-repl
+protobuf==4.21.12
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
 psutil==5.9.4
     # via edx-django-utils
 py==1.11.0
@@ -612,7 +627,9 @@ pynacl==1.5.0
 pyopenssl==22.1.0
     # via snowflake-connector-python
 pyparsing==3.0.9
-    # via packaging
+    # via
+    #   httplib2
+    #   packaging
 pyrsistent==0.19.3
     # via jsonschema
 pysocks==1.7.1
@@ -684,7 +701,7 @@ pyyaml==5.4.1
     #   edx-i18n-tools
 rcssmin==1.1.1
     # via django-compressor
-redis==4.4.1
+redis==4.4.2
     # via -r requirements/base.in
 requests==2.28.1
     # via
@@ -697,6 +714,7 @@ requests==2.28.1
     #   edx-analytics-data-api-client
     #   edx-drf-extensions
     #   edx-rest-api-client
+    #   google-api-core
     #   pyjwkest
     #   requests-file
     #   requests-oauthlib
@@ -760,6 +778,7 @@ six==1.16.0
     #   edx-sphinx-theme
     #   elasticsearch-dsl
     #   google-auth
+    #   google-auth-httplib2
     #   isodate
     #   jsonschema
     #   paramiko
@@ -868,7 +887,8 @@ uritemplate==4.1.1
     # via
     #   coreapi
     #   drf-yasg
-urllib3[socks]==1.26.13
+    #   google-api-python-client
+urllib3[socks]==1.26.14
     # via
     #   botocore
     #   docker
diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt
index fa45f86524d..83aa30c5ebf 100644
--- a/requirements/pip_tools.txt
+++ b/requirements/pip_tools.txt
@@ -4,20 +4,18 @@
 #
 #    pip-compile --output-file=requirements/pip_tools.txt requirements/pip_tools.in
 #
-build==0.9.0
+build==0.10.0
     # via pip-tools
 click==8.1.3
     # via pip-tools
 packaging==23.0
     # via build
-pep517==0.13.0
-    # via build
 pip-tools==6.12.1
     # via -r requirements/pip_tools.in
+pyproject-hooks==1.0.0
+    # via build
 tomli==2.0.1
-    # via
-    #   build
-    #   pep517
+    # via build
 wheel==0.38.4
     # via pip-tools
 
diff --git a/requirements/production.txt b/requirements/production.txt
index 70f8bf0f932..7995a56c7ea 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -39,9 +39,9 @@ beautifulsoup4==4.11.1
     #   taxonomy-connector
 billiard==3.6.4.0
     # via celery
-boto3==1.26.46
+boto3==1.26.48
     # via django-ses
-botocore==1.29.46
+botocore==1.29.48
     # via
     #   boto3
     #   s3transfer
@@ -338,16 +338,25 @@ future==0.18.2
     # via pyjwkest
 gevent==22.10.2
     # via -r requirements/production.in
+google-api-core==2.11.0
+    # via google-api-python-client
+google-api-python-client==2.72.0
+    # via -r requirements/base.in
 google-auth==2.16.0
     # via
+    #   google-api-core
+    #   google-api-python-client
+    #   google-auth-httplib2
     #   google-auth-oauthlib
     #   gspread
+google-auth-httplib2==0.1.0
+    # via
+    #   -r requirements/base.in
+    #   google-api-python-client
 google-auth-oauthlib==0.8.0
     # via gspread
-google-api-python-client==2.70.0
-    # via -r requirements/base.in
-google-auth-httplib2==0.1.0
-    # via google-api-python-client
+googleapis-common-protos==1.58.0
+    # via google-api-core
 greenlet==2.0.1
     # via gevent
 gspread==5.7.2
@@ -356,6 +365,10 @@ gunicorn==20.1.0
     # via -r requirements/production.in
 html2text==2020.1.16
     # via -r requirements/base.in
+httplib2==0.21.0
+    # via
+    #   google-api-python-client
+    #   google-auth-httplib2
 idna==3.4
     # via
     #   requests
@@ -408,8 +421,6 @@ openedx-events==4.1.0
     # via
     #   edx-event-bus-kafka
     #   taxonomy-connector
-oauth2client==4.1.3
-    # via -r requirements/base.in
 oscrypto==1.3.0
     # via snowflake-connector-python
 packaging==23.0
@@ -418,7 +429,7 @@ packaging==23.0
     #   drf-yasg
 pandas==1.5.2
     # via taxonomy-connector
-pbr==5.11.0
+pbr==5.11.1
     # via stevedore
 pillow==9.4.0
     # via
@@ -429,6 +440,10 @@ platformdirs==2.6.2
     # via zeep
 prompt-toolkit==3.0.36
     # via click-repl
+protobuf==4.21.12
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
 psutil==5.9.4
     # via edx-django-utils
 pyasn1==0.4.8
@@ -461,6 +476,8 @@ pynacl==1.5.0
     # via edx-django-utils
 pyopenssl==22.1.0
     # via snowflake-connector-python
+pyparsing==3.0.9
+    # via httplib2
 python-dateutil==2.8.2
     # via
     #   -r requirements/base.in
@@ -500,7 +517,7 @@ pyyaml==6.0
     #   edx-django-release-util
 rcssmin==1.1.1
     # via django-compressor
-redis==4.4.1
+redis==4.4.2
     # via -r requirements/base.in
 requests==2.28.1
     # via
@@ -511,6 +528,7 @@ requests==2.28.1
     #   edx-analytics-data-api-client
     #   edx-drf-extensions
     #   edx-rest-api-client
+    #   google-api-core
     #   pyjwkest
     #   requests-file
     #   requests-oauthlib
@@ -556,6 +574,7 @@ six==1.16.0
     #   edx-drf-extensions
     #   elasticsearch-dsl
     #   google-auth
+    #   google-auth-httplib2
     #   isodate
     #   pyjwkest
     #   python-dateutil
@@ -601,7 +620,8 @@ uritemplate==4.1.1
     # via
     #   coreapi
     #   drf-yasg
-urllib3==1.26.13
+    #   google-api-python-client
+urllib3==1.26.14
     # via
     #   botocore
     #   elasticsearch

From 098dd0114943b518ce99742028ec67040ac97136 Mon Sep 17 00:00:00 2001
From: Ali Akbar <52413434+Ali-D-Akbar@users.noreply.github.com>
Date: Tue, 17 Jan 2023 14:44:01 +0500
Subject: [PATCH 07/14] fix: add spaces in each word separately using
 rich_text_to_plain_text (#3760)

---
 .../apps/course_metadata/contentful_utils.py  |   9 +-
 .../contentful_utils/contentful_mock_data.py  | 103 ++++++++----------
 .../tests/test_contentful_utils.py            |  67 ++++++------
 .../apps/taxonomy_support/providers.py        |  11 +-
 4 files changed, 92 insertions(+), 98 deletions(-)

diff --git a/course_discovery/apps/course_metadata/contentful_utils.py b/course_discovery/apps/course_metadata/contentful_utils.py
index 8bfa7b6e779..cb2f7fdd6f7 100644
--- a/course_discovery/apps/course_metadata/contentful_utils.py
+++ b/course_discovery/apps/course_metadata/contentful_utils.py
@@ -150,7 +150,7 @@ def extract_plain_text_from_rich_text(rich_text_dict):
     text_list = []
     for key, value in rich_text_dict.items():
         if key == 'value':
-            text_list.append(value)
+            text_list.append(value.strip())
         elif isinstance(value, dict):  # recursive call if the value is a nested dict
             text_list += extract_plain_text_from_rich_text(value)
         elif isinstance(value, list):  # recursive call if the value is a nested list
@@ -164,7 +164,7 @@ def rich_text_to_plain_text(rich_text):
     Converts rich text from Contentful to plain text.
     Plain text resides in the dict with 'value' as dict key name.
     """
-    return ''.join(extract_plain_text_from_rich_text(rich_text))
+    return ' '.join(extract_plain_text_from_rich_text(rich_text))
 
 
 def get_modules_list(entry):
@@ -351,7 +351,10 @@ def fetch_and_transform_bootcamp_contentful_data():
     return transformed_bootcamp_data
 
 
-def get_aggregated_data_from_contentful_data(data, product_uuid):
+def aggregate_contentful_data(data, product_uuid):
+    """
+    Text data extracted from Contentful to be used in EMSI/Lightcast for product skills.
+    """
     if (data is None) or (product_uuid not in data):
         return None
 
diff --git a/course_discovery/apps/course_metadata/tests/contentful_utils/contentful_mock_data.py b/course_discovery/apps/course_metadata/tests/contentful_utils/contentful_mock_data.py
index 0f664c9c1b6..364ae638193 100644
--- a/course_discovery/apps/course_metadata/tests/contentful_utils/contentful_mock_data.py
+++ b/course_discovery/apps/course_metadata/tests/contentful_utils/contentful_mock_data.py
@@ -109,29 +109,25 @@ def __init__(self):
                     'content': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
                     'checkmarked_items': ['Lorem ipsum: dolor sit amet, consectetur adipiscing elit']
                 },
-                'faq_items': [
-                    {
-                        'question': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                        'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet,'
-                        ' consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem'
-                        ' ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, '
-                        'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
-                    },
-                ],
-                "featured_products": {
-                    "heading": "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-                    "introduction": "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-                    "product_list": [
-                        {
-                            "header": "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-                            "description": "Lorem ipsum: dolor sit amet, consectetur adipiscing elit"
-                        },
-                    ],
+                'featured_products': {
+                    'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                    'introduction': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                    'product_list': [{
+                        'header': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                        'description': 'Lorem ipsum: dolor sit amet, consectetur adipiscing elit'
+                    }]
+                },
+                'placement_about_section': {
+                    'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                    'body_text': 'Lorem ipsum: dolor sit amet, consectetur adipiscing elit'
                 },
-                "placement_about_section": {
-                    "heading": "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-                    "body_text": "Lorem ipsum: dolor sit amet, consectetur adipiscing elit",
-                }
+                'faq_items': [{
+                    'question': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                    'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                              ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                              ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                              ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                }]
             }
         }
 
@@ -232,9 +228,9 @@ def __init__(self):
             'test-uuid': {
                 'page_title': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
                 'subheading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                'hero_text_list': 'Lorem ipsum: dolor sit amet, consectetur adipiscing elitLorem ipsum: dolor sit amet,'
-                ' consectetur adipiscing elitLorem ipsum: dolor sit amet, consectetur adipiscing elitLorem'
-                ' ipsum: dolor sit amet, consectetur adipiscing elit',
+                'hero_text_list': 'Lorem ipsum: dolor sit amet, consectetur adipiscing elit Lorem ipsum: dolor sit '
+                                  'amet, consectetur adipiscing elit Lorem ipsum: dolor sit amet, consectetur '
+                                  'adipiscing elit Lorem ipsum: dolor sit amet, consectetur adipiscing elit',
                 'about_the_program': {
                     'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
                     'content': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
@@ -242,43 +238,40 @@ def __init__(self):
                 },
                 'blurb_1': {
                     'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                    'body': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet,'
-                            ' consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
-                            'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet,'
-                            ' consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
-                            'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                },
-                "blurb_2": {
-                    "heading": "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-                    "body": 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur'
-                            ' adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
-                },
-                'bootcamp_curriculum': {
-                    'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit '
-                    'amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur '
-                    'adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem '
-                    'ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet,'
-                    ' consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
+                    'body': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                            ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                            ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                            ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                            ' Lorem ipsum dolor sit amet, consectetur adipiscing elit'
                 },
-                'partnerships': {
+                'blurb_2': {
+                    'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+                    'body': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                            ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                }, 'bootcamp_curriculum': {
+                    'heading': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                               ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                               ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                               ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                               ' Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                }, 'partnerships': {
                     'heading_text': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                    'body_text': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem, ipsum, or dolorLorem '
-                                    'ipsum dolor sit amet, consectetur adipiscing elit',
-                },
-                'faq_items': [
+                    'body_text': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem , ipsum , or dolor'
+                                 ' Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                }, 'faq_items': [
                     {
                         'question': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                        'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet,'
-                                    ' consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem'
-                                    ' ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, '
-                                    'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
+                        'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                                  ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                                  ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                                  ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
                     },
                     {
                         'question': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
-                        'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, '
-                                    'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem'
-                                    ' ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur'
-                                    ' adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit'
+                        'answer': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                                  ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
+                                  ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,'
+                                  ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit'
                     }
                 ]
             }
diff --git a/course_discovery/apps/course_metadata/tests/test_contentful_utils.py b/course_discovery/apps/course_metadata/tests/test_contentful_utils.py
index cf31a8bc124..0393fd6113a 100644
--- a/course_discovery/apps/course_metadata/tests/test_contentful_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_contentful_utils.py
@@ -10,9 +10,8 @@
 from testfixtures import LogCapture
 
 from course_discovery.apps.course_metadata.contentful_utils import (
-    fetch_and_transform_bootcamp_contentful_data, fetch_and_transform_degree_contentful_data,
-    get_aggregated_data_from_contentful_data, get_contentful_cache_key, get_data_from_contentful,
-    rich_text_to_plain_text
+    aggregate_contentful_data, fetch_and_transform_bootcamp_contentful_data, fetch_and_transform_degree_contentful_data,
+    get_contentful_cache_key, get_data_from_contentful, rich_text_to_plain_text
 )
 from course_discovery.apps.course_metadata.tests.contentful_utils.contentful_mock_data import (
     MockContenfulDegreeResponse, MockContentfulBootcampResponse, create_contentful_entry
@@ -124,43 +123,43 @@ def test_transform_bootcamp_contentful_data(self, *args):
         self.assertDictEqual(
             transformed_data, mock_bootcamp_response.bootcamp_transformed_data)
 
-    def test_get_aggregated_data_from_contentful_data__bootcamp(self):
+    def test_get_aggregated_data_from_contentful__bootcamp(self):
         mock_bootcamp_response = MockContentfulBootcampResponse()
-        expected_data = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet, ' \
-                        'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem ' \
-                        'ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur ' \
-                        'adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit ' \
-                        'amet, consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit ' \
-                        'Lorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, ' \
-                        'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem ' \
-                        'ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur ' \
-                        'adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum: dolor ' \
-                        'sit amet, consectetur adipiscing elitLorem ipsum: dolor sit amet, consectetur adipiscing ' \
-                        'elitLorem ipsum: dolor sit amet, consectetur adipiscing elitLorem ipsum: dolor sit amet, ' \
-                        'consectetur adipiscing elit'
-
-        assert get_aggregated_data_from_contentful_data({}, 'uuid_123') is None
-        assert get_aggregated_data_from_contentful_data(
+        expected_data = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit ' \
+                        'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum: dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum: dolor sit amet, consectetur adipiscing elit Lorem ipsum: dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum: dolor sit amet, consectetur adipiscing elit'
+
+        assert aggregate_contentful_data({}, 'uuid_123') is None
+        assert aggregate_contentful_data(
             mock_bootcamp_response.bootcamp_transformed_data, 'no_uuid') is None
-        assert get_aggregated_data_from_contentful_data(
+        assert aggregate_contentful_data(
             mock_bootcamp_response.bootcamp_transformed_data, 'test-uuid') == expected_data
 
-    def test_get_aggregated_data_from_contentful_data__degree(self):
+    def test_get_aggregated_data_from_contentful__degree(self):
         mock_degree_response = MockContenfulDegreeResponse()
-        expected_data = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet, ' \
-                        'consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem ' \
-                        'ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit amet, consectetur ' \
-                        'adipiscing elitLorem ipsum dolor sit amet, consectetur adipiscing elitLorem ipsum dolor sit ' \
-                        'amet, consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit ' \
-                        'Lorem ipsum: dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet, ' \
-                        'consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ' \
-                        'ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum: dolor sit amet, consectetur ' \
-                        'adipiscing elit'
-
-        assert get_aggregated_data_from_contentful_data({}, 'uuid_123') is None
-        assert get_aggregated_data_from_contentful_data(
+        expected_data = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum: dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum dolor sit amet, consectetur adipiscing elit Lorem ipsum dolor sit amet,' \
+                        ' consectetur adipiscing elit Lorem ipsum dolor sit amet, consectetur adipiscing elit' \
+                        ' Lorem ipsum: dolor sit amet, consectetur adipiscing elit'
+
+        assert aggregate_contentful_data({}, 'uuid_123') is None
+        assert aggregate_contentful_data(
             mock_degree_response.degree_transformed_data, 'no_uuid') is None
-        assert get_aggregated_data_from_contentful_data(
+        assert aggregate_contentful_data(
             mock_degree_response.degree_transformed_data, 'test-uuid') == expected_data
 
     @mock.patch('course_discovery.apps.course_metadata.contentful_utils.get_data_from_contentful',
diff --git a/course_discovery/apps/taxonomy_support/providers.py b/course_discovery/apps/taxonomy_support/providers.py
index c7e51491016..80bcb866ce5 100644
--- a/course_discovery/apps/taxonomy_support/providers.py
+++ b/course_discovery/apps/taxonomy_support/providers.py
@@ -21,8 +21,7 @@
 from course_discovery.apps.core.api_client.lms import LMSAPIClient
 from course_discovery.apps.core.models import Partner
 from course_discovery.apps.course_metadata.contentful_utils import (
-    fetch_and_transform_bootcamp_contentful_data, fetch_and_transform_degree_contentful_data,
-    get_aggregated_data_from_contentful_data
+    aggregate_contentful_data, fetch_and_transform_bootcamp_contentful_data, fetch_and_transform_degree_contentful_data
 )
 from course_discovery.apps.course_metadata.models import Course, Program
 
@@ -45,7 +44,7 @@ def get_courses(course_ids):  # lint-amnesty, pylint: disable=arguments-differ
             'title': course.title,
             'short_description': course.short_description,
             'full_description': (
-                get_aggregated_data_from_contentful_data(contentful_data, str(course.uuid)) or course.full_description
+                aggregate_contentful_data(contentful_data, str(course.uuid)) or course.full_description
             ),
         } for course in courses]
 
@@ -64,7 +63,7 @@ def get_all_courses():  # lint-amnesty, pylint: disable=arguments-differ
                     'title': course.title,
                     'short_description': course.short_description,
                     'full_description': (
-                        get_aggregated_data_from_contentful_data(contentful_data, str(course.uuid)) or
+                        aggregate_contentful_data(contentful_data, str(course.uuid)) or
                         course.full_description
                     ),
                 }
@@ -87,7 +86,7 @@ def get_programs(program_ids):  # lint-amnesty, pylint: disable=arguments-differ
             'title': program.title,
             'subtitle': program.subtitle,
             'overview': (
-                get_aggregated_data_from_contentful_data(contentful_data, str(program.uuid)) or
+                aggregate_contentful_data(contentful_data, str(program.uuid)) or
                 program.overview
             ),
         } for program in programs]
@@ -106,7 +105,7 @@ def get_all_programs():  # lint-amnesty, pylint: disable=arguments-differ
                     'title': program.title,
                     'subtitle': program.subtitle,
                     'overview': (
-                        get_aggregated_data_from_contentful_data(contentful_data, str(program.uuid)) or
+                        aggregate_contentful_data(contentful_data, str(program.uuid)) or
                         program.overview
                     ),
                 }

From b217ba57ba7391644ef0c64f8d45fb700d30f5d7 Mon Sep 17 00:00:00 2001
From: aliadnan <ali.adnan@arbisoft.com>
Date: Tue, 17 Jan 2023 13:19:06 +0500
Subject: [PATCH 08/14] fix: fix google drive download url

---
 .../apps/course_metadata/tests/test_utils.py  | 26 +++++++++----------
 .../apps/course_metadata/utils.py             | 22 +++-------------
 2 files changed, 16 insertions(+), 32 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index bc54731f1ab..b6692a9a64c 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -726,7 +726,7 @@ def test_convert_svg_to_png_from_url(self, _svg2png_mock):
 class TestIsGoogleDriveUrl(TestCase):
     """Test is google drive url"""
     @ddt.data(
-        ('https://docs.google.com/uc?id=abcd123id', True),
+        ('https://drive.google.com/file/d/abcd12345/view?usp=sharing', True),
         ('https://example.com/image.jpg', False),
     )
     @ddt.unpack
@@ -758,9 +758,9 @@ def mock_image_response(self, status=200, body=None, content_type='image/jpeg',
         return image_url, body
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
-    def test_download_and_save_course_image_using_drive_link(self, mock_get_file_from_drive_link):
+    def test_download_and_save_course_image__using_drive_link(self, mock_get_file_from_drive_link):
         """ Verify that download_and_save_course_image will save image in course model using drive link """
-        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         course = CourseFactory(card_image_url=image_url, image=None)
         download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
         mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
@@ -770,7 +770,7 @@ def test_download_and_save_course_image_using_drive_link(self, mock_get_file_fro
         assert course.image is not None
 
     @responses.activate
-    def test_download_and_save_course_image_using_request_library(self):
+    def test_download_and_save_course_image__using_request_library(self):
         """ Verify that download_and_save_course_image will save image in course model using request response """
         image_url = 'https://example.com/image.jpg'
         course = CourseFactory(card_image_url=image_url, image=None)
@@ -783,9 +783,9 @@ def test_download_and_save_course_image_using_request_library(self):
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
     @responses.activate
-    def test_download_and_save_course_image_with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
+    def test_download_and_save_course_image__with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
         """ Verify that download_and_save_course_image will not save image in course model """
-        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         course = CourseFactory(card_image_url=image_url, image=None)
         download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
         mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
@@ -795,7 +795,7 @@ def test_download_and_save_course_image_with_invalid_content_type_using_drive_li
         assert not bool(course.image)
 
     @responses.activate
-    def test_download_and_save_course_image_with_invalid_content_type_using_request_library(self):
+    def test_download_and_save_course_image__with_invalid_content_type_using_request_library(self):
         """ Verify that download_and_save_course_image will not save image in course model """
         image_url = 'https://www.example.com/image.pdf'
         course = CourseFactory(card_image_url=image_url, image=None)
@@ -806,9 +806,9 @@ def test_download_and_save_course_image_with_invalid_content_type_using_request_
         assert not bool(course.image)
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
-    def test_download_and_save_program_image_using_drive_link(self, mock_get_file_from_drive_link):
+    def test_download_and_save_program_image__using_drive_link(self, mock_get_file_from_drive_link):
         """ Verify that download_and_save_program_image will save image in program model """
-        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
         download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
         mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
@@ -817,7 +817,7 @@ def test_download_and_save_program_image_using_drive_link(self, mock_get_file_fr
         assert program.card_image is not None
 
     @responses.activate
-    def test_download_and_save_program_image_using_request_library(self):
+    def test_download_and_save_program_image__using_request_library(self):
         """ Verify that download_and_save_program_image will save image in program model using request response """
         image_url = 'https://www.example.com'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
@@ -828,9 +828,9 @@ def test_download_and_save_program_image_using_request_library(self):
         assert program.card_image is not None
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
-    def test_download_and_save_program_image_with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
+    def test_download_and_save_program_image__with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
         """ Verify that download_and_save_program_image will not save image in program model using drive link """
-        image_url = 'https://docs.google.com/uc?id=abcd123id'
+        image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
         download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
         mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
@@ -839,7 +839,7 @@ def test_download_and_save_program_image_with_invalid_content_type_using_drive_l
         assert not bool(program.card_image)
 
     @responses.activate
-    def test_download_and_save_program_image_with_invalid_content_type_using_request_library(self):
+    def test_download_and_save_program_image__with_invalid_content_type_using_request_library(self):
         """ Verify that download_and_save_program_image will not save image in program model using request response """
         image_url = 'https://www.example.com/image.pdf'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index bdd25769822..310edfcd72a 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -1,7 +1,6 @@
 import datetime
 import logging
 import random
-import re
 import string
 import uuid
 from tempfile import NamedTemporaryFile
@@ -22,7 +21,7 @@
 
 from course_discovery.apps.core.models import SalesforceConfiguration
 from course_discovery.apps.core.utils import serialize_datetime
-from course_discovery.apps.course_metadata.constants import DRIVE_LINK_PATTERNS, IMAGE_TYPES
+from course_discovery.apps.course_metadata.constants import IMAGE_TYPES
 from course_discovery.apps.course_metadata.exceptions import (
     EcommerceSiteAPIClientException, MarketingSiteAPIClientException
 )
@@ -714,8 +713,6 @@ def download_and_save_course_image(course, image_url, data_field='image', header
     in the data field mentioned, defaulting to course card image.
     """
     try:
-        image_url = get_downloadable_url_from_drive_link(image_url)
-
         if is_google_drive_url(image_url):
             content_type, content = get_file_from_drive_link(image_url)
             extension = IMAGE_TYPES.get(content_type)
@@ -774,23 +771,12 @@ def convert_svg_to_png_from_url(image_url):
         return None
 
 
-def get_downloadable_url_from_drive_link(file_path):
-    """
-    Helper method to get the downloadable url from a drive link
-    """
-    URL = 'https://docs.google.com/uc?id={file_id}'
-    parsed_url = urlparse(file_path)
-    if parsed_url.hostname == 'drive.google.com':
-        file_id = parsed_url.path.split('/')[3]
-        return URL.format(file_id=file_id)
-    return file_path
-
-
 def is_google_drive_url(url):
     """
     Helper method to check if the file url is a drive url or not
     """
-    return any(re.match(pattern, url) for pattern in DRIVE_LINK_PATTERNS)
+    parsed_url = urlparse(url)
+    return parsed_url.hostname == 'drive.google.com'
 
 
 def download_and_save_program_image(program, image_url, data_field='image', headers=None):
@@ -800,8 +786,6 @@ def download_and_save_program_image(program, image_url, data_field='image', head
     """
     # TODO: refactor and merge program image download to use the same code as course image download
     try:
-        image_url = get_downloadable_url_from_drive_link(image_url)
-
         if is_google_drive_url(image_url):
             content_type, content = get_file_from_drive_link(image_url)
             extension = IMAGE_TYPES.get(content_type)

From 9f370a45e886e526512f93529828e23bf2ecbbff Mon Sep 17 00:00:00 2001
From: edX requirements bot
 <49161187+edx-requirements-bot@users.noreply.github.com>
Date: Wed, 18 Jan 2023 02:32:06 -0500
Subject: [PATCH 09/14] chore: python requirements update (#3762)

---
 requirements/docs.txt       |  8 ++++----
 requirements/local.txt      | 28 ++++++++++++----------------
 requirements/production.txt | 22 +++++++++-------------
 3 files changed, 25 insertions(+), 33 deletions(-)

diff --git a/requirements/docs.txt b/requirements/docs.txt
index 9a4e5e8f35b..f50b58d1712 100644
--- a/requirements/docs.txt
+++ b/requirements/docs.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/docs.txt requirements/docs.in
 #
-alabaster==0.7.12
+alabaster==0.7.13
     # via sphinx
 babel==2.11.0
     # via sphinx
@@ -12,7 +12,7 @@ certifi==2022.12.7
     # via
     #   elasticsearch
     #   requests
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
     # via requests
 django-elasticsearch-dsl @ git+https://github.com/django-es/django-elasticsearch-dsl.git@0e92e01c6ef74d2fe329965deee5f4b25da7ec87
     # via -r requirements/github.in
@@ -45,9 +45,9 @@ pygments==2.14.0
     # via sphinx
 python-dateutil==2.8.2
     # via elasticsearch-dsl
-pytz==2022.7
+pytz==2022.7.1
     # via babel
-requests==2.28.1
+requests==2.28.2
     # via sphinx
 six==1.16.0
     # via
diff --git a/requirements/local.txt b/requirements/local.txt
index 6b56c60e102..3039394d0f6 100644
--- a/requirements/local.txt
+++ b/requirements/local.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/local.txt requirements/local.in
 #
-alabaster==0.7.12
+alabaster==0.7.13
     # via sphinx
 algoliasearch==1.20.0
     # via
@@ -43,10 +43,6 @@ attrs==21.4.0
     #   semgrep
     #   trio
     #   zeep
-authlib==1.0.0rc1
-    # via
-    #   -c requirements/constraints.txt
-    #   simple-salesforce
 babel==2.11.0
     # via sphinx
 backoff==2.2.1
@@ -64,9 +60,9 @@ boltons==21.0.0
     #   face
     #   glom
     #   semgrep
-boto3==1.26.48
+boto3==1.26.50
     # via django-ses
-botocore==1.29.48
+botocore==1.29.50
     # via
     #   boto3
     #   s3transfer
@@ -76,7 +72,7 @@ cachetools==5.2.1
     # via google-auth
 cairocffi==1.4.0
     # via cairosvg
-cairosvg==2.5.2
+cairosvg==2.6.0
     # via -r requirements/base.in
 celery==5.2.7
     # via
@@ -140,7 +136,6 @@ coverage[toml]==7.0.5
     #   pytest-cov
 cryptography==38.0.4
     # via
-    #   authlib
     #   paramiko
     #   pyjwt
     #   pyopenssl
@@ -412,7 +407,7 @@ face==22.0.0
     # via glom
 factory-boy==3.2.1
     # via -r requirements/test.in
-faker==16.3.0
+faker==16.4.0
     # via factory-boy
 fastavro==1.7.0
     # via openedx-events
@@ -423,7 +418,7 @@ filelock==3.9.0
     #   virtualenv
 freezegun==1.2.2
     # via -r requirements/test.in
-future==0.18.2
+future==0.18.3
     # via pyjwkest
 glom==22.1.0
     # via semgrep
@@ -602,6 +597,7 @@ pyjwt[crypto]==2.6.0
     #   edx-auth-backends
     #   edx-drf-extensions
     #   edx-rest-api-client
+    #   simple-salesforce
     #   snowflake-connector-python
     #   social-auth-core
 pylint==2.15.10
@@ -634,7 +630,7 @@ pyrsistent==0.19.3
     # via jsonschema
 pysocks==1.7.1
     # via urllib3
-pytest==7.2.0
+pytest==7.2.1
     # via
     #   -r requirements/test.in
     #   pytest-cov
@@ -678,7 +674,7 @@ python-stdnum==1.18
     # via django-localflavor
 python3-openid==3.2.0
     # via social-auth-core
-pytz==2022.7
+pytz==2022.7.1
     # via
     #   -r requirements/base.in
     #   babel
@@ -703,7 +699,7 @@ rcssmin==1.1.1
     # via django-compressor
 redis==4.4.2
     # via -r requirements/base.in
-requests==2.28.1
+requests==2.28.2
     # via
     #   -r requirements/base.in
     #   algoliasearch
@@ -759,7 +755,7 @@ semgrep==0.102.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/test.in
-simple-salesforce==1.12.2
+simple-salesforce==1.12.3
     # via -r requirements/base.in
 six==1.16.0
     # via
@@ -907,7 +903,7 @@ virtualenv==20.17.1
     # via tox
 wcmatch==8.4.1
     # via semgrep
-wcwidth==0.2.5
+wcwidth==0.2.6
     # via prompt-toolkit
 webencodings==0.5.1
     # via
diff --git a/requirements/production.txt b/requirements/production.txt
index 7995a56c7ea..f857d3f82a0 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -27,10 +27,6 @@ attrs==22.2.0
     # via
     #   openedx-events
     #   zeep
-authlib==1.0.0rc1
-    # via
-    #   -c requirements/constraints.txt
-    #   simple-salesforce
 backoff==2.2.1
     # via -r requirements/base.in
 beautifulsoup4==4.11.1
@@ -39,9 +35,9 @@ beautifulsoup4==4.11.1
     #   taxonomy-connector
 billiard==3.6.4.0
     # via celery
-boto3==1.26.48
+boto3==1.26.50
     # via django-ses
-botocore==1.29.48
+botocore==1.29.50
     # via
     #   boto3
     #   s3transfer
@@ -49,7 +45,7 @@ cachetools==5.2.1
     # via google-auth
 cairocffi==1.4.0
     # via cairosvg
-cairosvg==2.5.2
+cairosvg==2.6.0
     # via -r requirements/base.in
 celery==5.2.7
     # via
@@ -97,7 +93,6 @@ coreschema==0.0.4
     #   drf-yasg
 cryptography==38.0.4
     # via
-    #   authlib
     #   pyjwt
     #   pyopenssl
     #   snowflake-connector-python
@@ -334,7 +329,7 @@ fastavro==1.7.0
     # via openedx-events
 filelock==3.9.0
     # via snowflake-connector-python
-future==0.18.2
+future==0.18.3
     # via pyjwkest
 gevent==22.10.2
     # via -r requirements/production.in
@@ -468,6 +463,7 @@ pyjwt[crypto]==2.6.0
     #   edx-auth-backends
     #   edx-drf-extensions
     #   edx-rest-api-client
+    #   simple-salesforce
     #   snowflake-connector-python
     #   social-auth-core
 pymongo==3.13.0
@@ -498,7 +494,7 @@ python-stdnum==1.18
     # via django-localflavor
 python3-openid==3.2.0
     # via social-auth-core
-pytz==2022.7
+pytz==2022.7.1
     # via
     #   -r requirements/base.in
     #   celery
@@ -519,7 +515,7 @@ rcssmin==1.1.1
     # via django-compressor
 redis==4.4.2
     # via -r requirements/base.in
-requests==2.28.1
+requests==2.28.2
     # via
     #   -r requirements/base.in
     #   algoliasearch
@@ -558,7 +554,7 @@ s3transfer==0.6.0
     # via boto3
 semantic-version==2.10.0
     # via edx-drf-extensions
-simple-salesforce==1.12.2
+simple-salesforce==1.12.3
     # via -r requirements/base.in
 six==1.16.0
     # via
@@ -632,7 +628,7 @@ vine==5.0.0
     #   amqp
     #   celery
     #   kombu
-wcwidth==0.2.5
+wcwidth==0.2.6
     # via prompt-toolkit
 webencodings==0.5.1
     # via

From 9dc64916686d73f97b11bba438d6e609124e7e49 Mon Sep 17 00:00:00 2001
From: Muhammad Afaq Shuaib <78806673+AfaqShuaib09@users.noreply.github.com>
Date: Wed, 18 Jan 2023 14:27:13 +0500
Subject: [PATCH 10/14] fix: fix: add only allowed attribute to anchor tag
 (#3763)

Co-authored-by: afaq.shuaib <afaq.shuaib09@arbisoft.com>
---
 course_discovery/apps/course_metadata/constants.py        | 2 ++
 course_discovery/apps/course_metadata/tests/test_utils.py | 4 ++++
 course_discovery/apps/course_metadata/utils.py            | 7 ++++---
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/course_discovery/apps/course_metadata/constants.py b/course_discovery/apps/course_metadata/constants.py
index 3dc91c41b33..ac8e78e0983 100644
--- a/course_discovery/apps/course_metadata/constants.py
+++ b/course_discovery/apps/course_metadata/constants.py
@@ -15,6 +15,8 @@
     'image/svg+xml': 'svg'  # SVG image will be converted into PNG, not stored as SVG
 }
 
+ALLOWED_ANCHOR_TAG_ATTRIBUTES = ['href', 'title', 'target', 'rel']
+
 DRIVE_LINK_PATTERNS = [r"https://docs\.google\.com/uc\?id=\w+",
                        r"https://drive\.google\.com/file/d/\w+/view?usp=sharing"]
 
diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index b6692a9a64c..ddb280e4c9f 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -626,6 +626,10 @@ class UtilsTests(TestCase):
         # pylint: disable=line-too-long
         ('<a href="https://yahoo.com" target="_blank" rel="noopener">link</a>', '<p><a href="https://yahoo.com" rel="noopener" target="_blank">link</a></p>'),
 
+        # Make sure not to add data-ol-has-click-handler attribute to anchor tags if they are in attributes list
+        # pylint: disable=line-too-long
+        ('<p>please visit this <a title="less go" href="https://google.com" data-ol-has-click-handler="true">link</a></p>', '<p>please visit this <a href="https://google.com" title="less go">link</a></p>'),
+
         # And make sure we strip what we should
         ('<p class="float">Class</p>', '<p>Class</p>'),
         ('<p style="float">Inline Style</p>', '<p>Inline Style</p>'),
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index 310edfcd72a..4065cbdc1df 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -21,7 +21,7 @@
 
 from course_discovery.apps.core.models import SalesforceConfiguration
 from course_discovery.apps.core.utils import serialize_datetime
-from course_discovery.apps.course_metadata.constants import IMAGE_TYPES
+from course_discovery.apps.course_metadata.constants import ALLOWED_ANCHOR_TAG_ATTRIBUTES, IMAGE_TYPES
 from course_discovery.apps.course_metadata.exceptions import (
     EcommerceSiteAPIClientException, MarketingSiteAPIClientException
 )
@@ -662,11 +662,12 @@ def handle_tag(self, tag, attrs, start):
                 self.in_lang_span = False
 
         if tag == 'a':
-            # override the default behavior of html2text to include all attributes from attr_dict for <a> tags
+            # override the default behavior of html2text to include only allowed tags from attr_dict for <a> tags
             # because by default it only includes the href and title attributes
             if attrs and start and 'href' in dict(attrs):
                 self.outtextf('<a')
-                for attr, value in dict(attrs).items():
+                filtered_attrs_list = [(attr, value) for attr, value in dict(attrs).items() if attr in ALLOWED_ANCHOR_TAG_ATTRIBUTES]  # pylint: disable=line-too-long
+                for attr, value in filtered_attrs_list:
                     self.outtextf(f' {attr}="{value}"')
                 self.outtextf('>')
             if not start:

From 4af81c171f50ea804878eb0aa00da0e90fbd8422 Mon Sep 17 00:00:00 2001
From: Ali Akbar <52413434+Ali-D-Akbar@users.noreply.github.com>
Date: Wed, 18 Jan 2023 15:09:26 +0500
Subject: [PATCH 11/14] fix: set default to null for course_term_override
 (#3765)

---
 ..._additionalmetadata_course_term_override.py | 18 ++++++++++++++++++
 .../apps/course_metadata/models.py             |  2 ++
 2 files changed, 20 insertions(+)
 create mode 100644 course_discovery/apps/course_metadata/migrations/0309_alter_additionalmetadata_course_term_override.py

diff --git a/course_discovery/apps/course_metadata/migrations/0309_alter_additionalmetadata_course_term_override.py b/course_discovery/apps/course_metadata/migrations/0309_alter_additionalmetadata_course_term_override.py
new file mode 100644
index 00000000000..fa3ec214fab
--- /dev/null
+++ b/course_discovery/apps/course_metadata/migrations/0309_alter_additionalmetadata_course_term_override.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-01-17 20:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('course_metadata', '0308_auto_20230111_1300'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='additionalmetadata',
+            name='course_term_override',
+            field=models.CharField(blank=True, default=None, help_text='This field allows for override the default course term', max_length=20, null=True, verbose_name='Course override'),
+        ),
+    ]
diff --git a/course_discovery/apps/course_metadata/models.py b/course_discovery/apps/course_metadata/models.py
index 3fc257bad49..6a6158cb94e 100644
--- a/course_discovery/apps/course_metadata/models.py
+++ b/course_discovery/apps/course_metadata/models.py
@@ -756,6 +756,8 @@ class AdditionalMetadata(TimeStampedModel):
         verbose_name=_('Course override'),
         help_text=_('This field allows for override the default course term'),
         blank=True,
+        null=True,
+        default=None,
     )
     product_meta = models.OneToOneField(
         ProductMeta,

From aa5d66a7f82e0e214b67206dec5965ab2eef5d3f Mon Sep 17 00:00:00 2001
From: edX requirements bot <devops+edx-requirements-bot@edx.org>
Date: Wed, 18 Jan 2023 11:27:32 -0500
Subject: [PATCH 12/14] chore: python requirements update

---
 requirements/docs.txt       |  4 ++--
 requirements/local.txt      | 14 +++++++-------
 requirements/production.txt | 10 +++++-----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/requirements/docs.txt b/requirements/docs.txt
index f50b58d1712..f46d8d35a9b 100644
--- a/requirements/docs.txt
+++ b/requirements/docs.txt
@@ -18,7 +18,7 @@ django-elasticsearch-dsl @ git+https://github.com/django-es/django-elasticsearch
     # via -r requirements/github.in
 docutils==0.19
     # via sphinx
-edx-sphinx-theme==3.0.0
+edx-sphinx-theme==3.1.0
     # via -r requirements/docs.in
 elasticsearch==7.13.4
     # via
@@ -37,7 +37,7 @@ importlib-metadata==6.0.0
     # via sphinx
 jinja2==3.1.2
     # via sphinx
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via jinja2
 packaging==23.0
     # via sphinx
diff --git a/requirements/local.txt b/requirements/local.txt
index 3039394d0f6..4f85ff2c4df 100644
--- a/requirements/local.txt
+++ b/requirements/local.txt
@@ -60,9 +60,9 @@ boltons==21.0.0
     #   face
     #   glom
     #   semgrep
-boto3==1.26.50
+boto3==1.26.51
     # via django-ses
-botocore==1.29.50
+botocore==1.29.51
     # via
     #   boto3
     #   s3transfer
@@ -363,7 +363,7 @@ edx-drf-extensions==8.2.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-edx-event-bus-kafka==3.5.1
+edx-event-bus-kafka==3.6.0
     # via -r requirements/base.in
 edx-i18n-tools==0.9.2
     # via -r requirements/local.in
@@ -380,7 +380,7 @@ edx-rest-api-client==5.5.0
     # via
     #   -r requirements/base.in
     #   taxonomy-connector
-edx-sphinx-theme==3.0.0
+edx-sphinx-theme==3.1.0
     # via -r requirements/docs.in
 edx-toggles==5.0.0
     # via edx-event-bus-kafka
@@ -407,7 +407,7 @@ face==22.0.0
     # via glom
 factory-boy==3.2.1
     # via -r requirements/test.in
-faker==16.4.0
+faker==16.6.0
     # via factory-boy
 fastavro==1.7.0
     # via openedx-events
@@ -424,7 +424,7 @@ glom==22.1.0
     # via semgrep
 google-api-core==2.11.0
     # via google-api-python-client
-google-api-python-client==2.72.0
+google-api-python-client==2.73.0
     # via -r requirements/base.in
 google-auth==2.16.0
     # via
@@ -502,7 +502,7 @@ lxml==4.9.2
     #   zeep
 markdown==3.4.1
     # via -r requirements/base.in
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via jinja2
 mccabe==0.7.0
     # via pylint
diff --git a/requirements/production.txt b/requirements/production.txt
index f857d3f82a0..60d9ddb26b0 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -35,9 +35,9 @@ beautifulsoup4==4.11.1
     #   taxonomy-connector
 billiard==3.6.4.0
     # via celery
-boto3==1.26.50
+boto3==1.26.51
     # via django-ses
-botocore==1.29.50
+botocore==1.29.51
     # via
     #   boto3
     #   s3transfer
@@ -297,7 +297,7 @@ edx-drf-extensions==8.2.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-edx-event-bus-kafka==3.5.1
+edx-event-bus-kafka==3.6.0
     # via -r requirements/base.in
 edx-opaque-keys[django]==2.3.0
     # via
@@ -335,7 +335,7 @@ gevent==22.10.2
     # via -r requirements/production.in
 google-api-core==2.11.0
     # via google-api-python-client
-google-api-python-client==2.72.0
+google-api-python-client==2.73.0
     # via -r requirements/base.in
 google-auth==2.16.0
     # via
@@ -398,7 +398,7 @@ lxml==4.9.2
     #   zeep
 markdown==3.4.1
     # via -r requirements/base.in
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via jinja2
 mysqlclient==2.1.1
     # via -r requirements/production.in

From c21519084c4d182867d549b150b1ce907239f032 Mon Sep 17 00:00:00 2001
From: Muhammad Afaq Shuaib <78806673+AfaqShuaib09@users.noreply.github.com>
Date: Mon, 23 Jan 2023 17:46:07 +0500
Subject: [PATCH 13/14] fix: Unbond Local errror for response variable (#3768)

Co-authored-by: afaq.shuaib <afaq.shuaib09@arbisoft.com>
---
 .../apps/course_metadata/tests/test_utils.py  | 96 ++++++++++++++-----
 .../apps/course_metadata/utils.py             |  6 +-
 2 files changed, 74 insertions(+), 28 deletions(-)

diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py
index ddb280e4c9f..afbe2d0e393 100644
--- a/course_discovery/apps/course_metadata/tests/test_utils.py
+++ b/course_discovery/apps/course_metadata/tests/test_utils.py
@@ -739,7 +739,6 @@ def test_is_google_drive_url(self, url, expected):
         assert is_google_drive_url(url) is expected
 
 
-@ddt.ddt
 class TestDownloadAndSaveImage(TestCase):
     """ Test to download and save image """
 
@@ -766,24 +765,27 @@ def test_download_and_save_course_image__using_drive_link(self, mock_get_file_fr
         """ Verify that download_and_save_course_image will save image in course model using drive link """
         image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         course = CourseFactory(card_image_url=image_url, image=None)
-        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.return_value = ('image/jpeg', self.IMG_CONTENT)
+        assert download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None) is True
         mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
-        mock_get_file_from_drive_link = mock.Mock()
-        mock_get_file_from_drive_link.return_value = (self.IMG_CONTENT, 'image/jpeg')
+        course.refresh_from_db()
         assert course.card_image_url == image_url
         assert course.image is not None
+        assert course.image.read() == self.IMG_CONTENT
+        assert str(course.uuid) in course.image.name
 
     @responses.activate
     def test_download_and_save_course_image__using_request_library(self):
         """ Verify that download_and_save_course_image will save image in course model using request response """
         image_url = 'https://example.com/image.jpg'
         course = CourseFactory(card_image_url=image_url, image=None)
-        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
         image_url, content = self.mock_image_response()
-        response = requests.get('https://example.com/image.jpg', timeout=5)
-        assert response.content == content
+        assert download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None) is True
+        course.refresh_from_db()
         assert course.card_image_url == image_url
         assert course.image is not None
+        assert course.image.read() == content
+        assert str(course.uuid) in course.image.name
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
     @responses.activate
@@ -791,11 +793,12 @@ def test_download_and_save_course_image__with_invalid_content_type_using_drive_l
         """ Verify that download_and_save_course_image will not save image in course model """
         image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         course = CourseFactory(card_image_url=image_url, image=None)
-        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.return_value = ('text/plain', b'invalid')
+        assert download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None) is False
         mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
-        mock_get_file_from_drive_link = mock.Mock()
-        mock_get_file_from_drive_link.return_value = (b'invalid', 'text/plain')
+        course.refresh_from_db()
         assert course.card_image_url == image_url
+        assert course.image.name == ''
         assert not bool(course.image)
 
     @responses.activate
@@ -803,10 +806,11 @@ def test_download_and_save_course_image__with_invalid_content_type_using_request
         """ Verify that download_and_save_course_image will not save image in course model """
         image_url = 'https://www.example.com/image.pdf'
         course = CourseFactory(card_image_url=image_url, image=None)
-        download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None)
-        image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
-        response = requests.get('https://www.example.com/image.pdf', timeout=5)
-        assert response.content == content
+        image_url, _ = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
+        assert download_and_save_course_image(course, course.card_image_url, data_field='image', headers=None) is False
+        course.refresh_from_db()
+        assert course.card_image_url == image_url
+        assert course.image.name == ''
         assert not bool(course.image)
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
@@ -814,32 +818,39 @@ def test_download_and_save_program_image__using_drive_link(self, mock_get_file_f
         """ Verify that download_and_save_program_image will save image in program model """
         image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
-        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        mock_get_file_from_drive_link.return_value = ('image/jpeg', self.IMG_CONTENT)
+        assert download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None) is True  # pylint: disable=line-too-long
         mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
-        mock_get_file_from_drive_link.return_value = (self.IMG_CONTENT, 'image/jpeg')
+        program.refresh_from_db()
         assert program.card_image_url == image_url
         assert program.card_image is not None
+        assert program.card_image.read() == self.IMG_CONTENT
+        assert str(program.uuid) in program.card_image.name
 
     @responses.activate
     def test_download_and_save_program_image__using_request_library(self):
         """ Verify that download_and_save_program_image will save image in program model using request response """
-        image_url = 'https://www.example.com'
+        image_url = 'https://example.com/image.jpg'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
-        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
-        image_url, content = self.mock_image_response()
-        response = requests.get('https://example.com/image.jpg', timeout=5)
-        assert response.content == content
+        image_url, content = self.mock_image_response(url=image_url)
+        assert download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None) is True  # pylint: disable=line-too-long
+        program.refresh_from_db()
+        assert program.card_image_url == image_url
         assert program.card_image is not None
+        assert program.card_image.read() == content
+        assert str(program.uuid) in program.card_image.name
 
     @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
     def test_download_and_save_program_image__with_invalid_content_type_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
         """ Verify that download_and_save_program_image will not save image in program model using drive link """
         image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
-        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
+        assert download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None) is False  # pylint: disable=line-too-long
         mock_get_file_from_drive_link.assert_called_once_with(program.card_image_url)
         mock_get_file_from_drive_link.return_value = (b'invalid', 'text/plain')
+        program.refresh_from_db()
         assert program.card_image_url == image_url
+        assert program.card_image.name == ''
         assert not bool(program.card_image)
 
     @responses.activate
@@ -847,8 +858,41 @@ def test_download_and_save_program_image__with_invalid_content_type_using_reques
         """ Verify that download_and_save_program_image will not save image in program model using request response """
         image_url = 'https://www.example.com/image.pdf'
         program = ProgramFactory(card_image_url=image_url, card_image=None)
-        download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None)
-        image_url, content = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)  # pylint: disable=line-too-long
-        response = requests.get('https://www.example.com/image.pdf', timeout=5)
-        assert response.content == content
+        image_url, _ = self.mock_image_response(status=200, body=b'invalid', content_type='text/plain', url=image_url)
+        assert download_and_save_program_image(program, program.card_image_url, data_field='image', headers=None) is False  # pylint: disable=line-too-long
+        program.refresh_from_db()
+        assert program.card_image_url == image_url
+        assert program.card_image.name == ''
         assert not bool(program.card_image)
+
+    @responses.activate
+    def test_download_and_save_course_image__for_organization_logo_override_using_request_library(self):
+        """
+        Verify that download_and_save_course_image will save image in course model
+        for organization_logo_override using request response
+        """
+        image_url = 'https://www.example.com/image.jpg'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        image_url, content = self.mock_image_response(url=image_url)
+        assert download_and_save_course_image(course, course.card_image_url, data_field='organization_logo_override', headers=None) is True  # pylint: disable=line-too-long
+        course.refresh_from_db()
+        assert course.card_image_url == image_url
+        assert course.organization_logo_override.read() == content
+        assert course.organization_logo_override is not None
+        assert str(course.uuid) in course.organization_logo_override.name
+
+    @mock.patch('course_discovery.apps.course_metadata.utils.get_file_from_drive_link')
+    def test_download_and_save_course_image__for_organization_logo_override_using_drive_link(self, mock_get_file_from_drive_link):  # pylint: disable=line-too-long
+        """
+        Verify that download_and_save_course_image will save image in course model
+        for organization_logo_override using drive link
+        """
+        image_url = 'https://drive.google.com/file/d/abcd12345/view?usp=sharing'
+        course = CourseFactory(card_image_url=image_url, image=None)
+        mock_get_file_from_drive_link.return_value = ('image/png', self.IMG_CONTENT)
+        assert download_and_save_course_image(course, course.card_image_url, data_field='organization_logo_override', headers=None) is True  # pylint: disable=line-too-long
+        mock_get_file_from_drive_link.assert_called_once_with(course.card_image_url)
+        course.refresh_from_db()
+        assert course.organization_logo_override is not None
+        assert course.organization_logo_override.read() == self.IMG_CONTENT
+        assert str(course.uuid) in course.organization_logo_override.name
diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py
index 4065cbdc1df..9d420961cd8 100644
--- a/course_discovery/apps/course_metadata/utils.py
+++ b/course_discovery/apps/course_metadata/utils.py
@@ -666,7 +666,9 @@ def handle_tag(self, tag, attrs, start):
             # because by default it only includes the href and title attributes
             if attrs and start and 'href' in dict(attrs):
                 self.outtextf('<a')
-                filtered_attrs_list = [(attr, value) for attr, value in dict(attrs).items() if attr in ALLOWED_ANCHOR_TAG_ATTRIBUTES]  # pylint: disable=line-too-long
+                filtered_attrs_list = [
+                    (attr, value) for attr, value in dict(attrs).items() if attr in ALLOWED_ANCHOR_TAG_ATTRIBUTES
+                ]
                 for attr, value in filtered_attrs_list:
                     self.outtextf(f' {attr}="{value}"')
                 self.outtextf('>')
@@ -737,7 +739,7 @@ def download_and_save_course_image(course, image_url, data_field='image', header
             if data_field == 'image':
                 course.image.save(filename, ContentFile(content))
             elif data_field == 'organization_logo_override':
-                image_file = ContentFile(response.content)
+                image_file = ContentFile(content)
                 if extension == 'svg':
                     filename = '{uuid}.png'.format(uuid=str(course.uuid))
                     image_file = convert_svg_to_png_from_url(image_url)

From 7857791e0d9359a790eb3a1d4abf388c71d936bc Mon Sep 17 00:00:00 2001
From: edX requirements bot <devops+edx-requirements-bot@edx.org>
Date: Sun, 22 Jan 2023 19:22:37 -0500
Subject: [PATCH 14/14] chore: python requirements update

---
 requirements/local.txt      | 19 +++++++++----------
 requirements/production.txt | 12 ++++++------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/requirements/local.txt b/requirements/local.txt
index 4f85ff2c4df..ce4673eace1 100644
--- a/requirements/local.txt
+++ b/requirements/local.txt
@@ -23,7 +23,7 @@ asn1crypto==1.5.1
     # via
     #   oscrypto
     #   snowflake-connector-python
-astroid==2.13.2
+astroid==2.13.3
     # via
     #   pylint
     #   pylint-celery
@@ -60,15 +60,15 @@ boltons==21.0.0
     #   face
     #   glom
     #   semgrep
-boto3==1.26.51
+boto3==1.26.54
     # via django-ses
-botocore==1.29.51
+botocore==1.29.54
     # via
     #   boto3
     #   s3transfer
 bracex==2.3.post1
     # via wcmatch
-cachetools==5.2.1
+cachetools==5.3.0
     # via google-auth
 cairocffi==1.4.0
     # via cairosvg
@@ -213,7 +213,7 @@ django-choices==1.7.2
     # via
     #   -r requirements/base.in
     #   taxonomy-connector
-django-compressor==4.3
+django-compressor==4.3.1
     # via
     #   -r requirements/base.in
     #   django-libsass
@@ -363,7 +363,7 @@ edx-drf-extensions==8.2.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-edx-event-bus-kafka==3.6.0
+edx-event-bus-kafka==3.6.1
     # via -r requirements/base.in
 edx-i18n-tools==0.9.2
     # via -r requirements/local.in
@@ -535,9 +535,9 @@ packaging==21.3
     #   semgrep
     #   sphinx
     #   tox
-pandas==1.5.2
+pandas==1.5.3
     # via taxonomy-connector
-paramiko==2.12.0
+paramiko==3.0.0
     # via docker
 path==16.6.0
     # via edx-i18n-tools
@@ -658,7 +658,7 @@ python-dateutil==2.8.2
     #   faker
     #   freezegun
     #   pandas
-python-dotenv==0.21.0
+python-dotenv==0.21.1
     # via docker-compose
 python-lsp-jsonrpc==1.0.0
     # via semgrep
@@ -777,7 +777,6 @@ six==1.16.0
     #   google-auth-httplib2
     #   isodate
     #   jsonschema
-    #   paramiko
     #   pyjwkest
     #   python-dateutil
     #   python-memcached
diff --git a/requirements/production.txt b/requirements/production.txt
index 60d9ddb26b0..ad2c32ab13e 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -35,13 +35,13 @@ beautifulsoup4==4.11.1
     #   taxonomy-connector
 billiard==3.6.4.0
     # via celery
-boto3==1.26.51
+boto3==1.26.54
     # via django-ses
-botocore==1.29.51
+botocore==1.29.54
     # via
     #   boto3
     #   s3transfer
-cachetools==5.2.1
+cachetools==5.3.0
     # via google-auth
 cairocffi==1.4.0
     # via cairosvg
@@ -159,7 +159,7 @@ django-choices==1.7.2
     # via
     #   -r requirements/base.in
     #   taxonomy-connector
-django-compressor==4.3
+django-compressor==4.3.1
     # via
     #   -r requirements/base.in
     #   django-libsass
@@ -297,7 +297,7 @@ edx-drf-extensions==8.2.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-edx-event-bus-kafka==3.6.0
+edx-event-bus-kafka==3.6.1
     # via -r requirements/base.in
 edx-opaque-keys[django]==2.3.0
     # via
@@ -422,7 +422,7 @@ packaging==23.0
     # via
     #   django-nine
     #   drf-yasg
-pandas==1.5.2
+pandas==1.5.3
     # via taxonomy-connector
 pbr==5.11.1
     # via stevedore