This is the first card of the exploration.
', + }, + } + ), + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'End State', + 'new_value': { + 'content_id': 'content_3', + 'html': 'This is the last card of the exploration.
', + }, + } + ), + ], + 'Changes content.', + ) def test_should_be_able_to_regenerate_voiceovers(self) -> None: self.login(self.VOICE_ARTIST_EMAIL) @@ -290,7 +326,7 @@ def test_should_be_able_to_regenerate_voiceovers(self) -> None: 'language_accent_code': 'en-US', 'state_name': 'Introduction', 'content_id': 'content_0', - 'exploration_version': 1, + 'exploration_version': 2, } handler_url = '/regenerate_automatic_voiceover/%s' % self.exploration.id @@ -362,31 +398,23 @@ def mock_defer( function_id: str, queue_name: str, exploration_id: str, - exploration_title: str, exploration_version: int, - committer_id: str, - datetime_str: str, ) -> None: deferred_calls.append( { 'function_id': function_id, 'queue_name': queue_name, 'exploration_id': exploration_id, - 'exploration_title': exploration_title, 'exploration_version': exploration_version, - 'committer_id': committer_id, - 'datetime_str': datetime_str, } ) exploration_id = self.exploration.id exploration_version = self.exploration.version - exploration_title = self.exploration.title - handler_url = '/regenerate_voiceover_on_exp_update/%s/%s/%s' % ( + handler_url = '/regenerate_voiceover_on_exp_update/%s/%s' % ( exploration_id, exploration_version, - exploration_title, ) with ( @@ -411,9 +439,7 @@ def mock_defer( self.assertEqual(args['function_id'], expected_func_name) self.assertEqual(args['queue_name'], 'voiceover-regeneration') self.assertEqual(args['exploration_id'], exploration_id) - self.assertEqual(args['exploration_title'], exploration_title) self.assertEqual(args['exploration_version'], exploration_version) - self.assertEqual(args['committer_id'], feconf.SYSTEM_COMMITTER_ID) self.logout() @@ -445,7 +471,7 @@ def test_get_automatic_voiceover_regeneration_records(self) -> None: queue_name, task_id, ) - function_id = 'delete_exps_from_user_models' + function_id = 'regenerate_voiceovers_on_exploration_update' taskqueue_services.create_new_cloud_task_model( new_model_id, task_name, function_id @@ -470,7 +496,7 @@ def test_get_automatic_voiceover_regeneration_records(self) -> None: ) self.assertEqual( json_response['automatic_voiceover_regeneration_records'], - [cloud_task_run.to_dict()], + [cloud_task_run.to_dict_with_timezone_info()], ) self.logout() @@ -497,14 +523,14 @@ def test_get_automatic_voiceover_regeneration_status(self) -> None: 'en-US': {'content_0': 'SUCCEEDED', 'content_1': 'SUCCEEDED'} } voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping( + cloud_task_domain.VoiceoverRegenerationJob( exploration_id, task_run_id, language_accent_to_content_status_map, ) ) - voiceover_cloud_task_services.save_voiceover_regeneration_task_run_mapping( + voiceover_cloud_task_services.save_voiceover_regeneration_job( voiceover_regeneration_task_mapping ) exploration = exp_domain.Exploration.create_default_exploration( @@ -711,7 +737,6 @@ def test_regenerate_voiceovers_on_exploration_added_to_topic(self) -> None: cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() function_id = cloud_task_runs[0].function_id task_run_id = cloud_task_runs[0].task_run_id - created_on_time_str = cloud_task_runs[0].created_on.isoformat() # Verifying that a Cloud Task run is created to regenerate the # voiceovers. @@ -736,11 +761,22 @@ def test_regenerate_voiceovers_on_exploration_added_to_topic(self) -> None: # via a deferred job. voiceover_services.regenerate_voiceovers_on_exploration_added_to_topic( self.exploration_id, - created_on_time_str, - feconf.SYSTEM_COMMITTER_ID, task_run_id, ) + updated_cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() + + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + child_task_run_id = cloud_run.task_run_id + + voiceover_services.regenerate_voiceovers_for_batch_contents( + self.exploration_id, task_run_id, child_task_run_id + ) + entity_voiceovers = ( voiceover_services.get_entity_voiceovers_for_given_exploration( self.exploration_id, 'exploration', 2 @@ -753,10 +789,8 @@ def test_regenerate_voiceovers_on_exploration_added_to_topic(self) -> None: self.assertEqual(entity_voiceovers[0].entity_id, self.exploration_id) self.assertEqual(entity_voiceovers[0].language_accent_code, 'en-US') - # The exploration contains two non-empty contents, content_0 and - # content_3, that are voiceovered automatically. self.assertListEqual( - ['content_0', 'content_3'], + ['content_0', 'default_outcome_1', 'ca_placeholder_2', 'content_3'], list(entity_voiceovers[0].voiceovers_mapping.keys()), ) @@ -771,6 +805,8 @@ def test_regenerate_voiceovers_on_exploration_added_to_topic(self) -> None: ] automated_voiceovers_audio_offsets_msecs = { 'content_0': dummy_audio_offset, + 'default_outcome_1': dummy_audio_offset, + 'ca_placeholder_2': dummy_audio_offset, 'content_3': dummy_audio_offset, } @@ -825,7 +861,6 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() function_id = cloud_task_runs[0].function_id task_run_id = cloud_task_runs[0].task_run_id - created_on_time_str = cloud_task_runs[0].created_on.isoformat() # Verifying that a Cloud Task run is created to regenerate the # voiceovers. @@ -850,11 +885,22 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: # via a deferred job. voiceover_services.regenerate_voiceovers_on_exploration_added_to_topic( self.exploration_id, - created_on_time_str, - feconf.SYSTEM_COMMITTER_ID, task_run_id, ) + updated_cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() + + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + child_task_run_id = cloud_run.task_run_id + + voiceover_services.regenerate_voiceovers_for_batch_contents( + self.exploration_id, task_run_id, child_task_run_id + ) + entity_voiceovers = ( voiceover_services.get_entity_voiceovers_for_given_exploration( self.exploration_id, 'exploration', 2 @@ -893,10 +939,9 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: # Simulating the frontend request that triggers voiceover regeneration # after an exploration update via a deferred job. - handler_url = '/regenerate_voiceover_on_exp_update/%s/%s/%s' % ( + handler_url = '/regenerate_voiceover_on_exp_update/%s/%s' % ( self.exploration_id, updated_exp.version, - updated_exp.title, ) csrf_token = self.get_new_csrf_token() self.post_json(handler_url, {}, csrf_token=csrf_token) @@ -909,10 +954,9 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: # Updating a curated exploration triggers voiceover regeneration via # the Cloud Task service, confirming that a deferred request exists in # the model. - cloud_task_run = cloud_task_runs[1] + cloud_task_run = cloud_task_runs[2] function_id = cloud_task_run.function_id task_run_id = cloud_task_run.task_run_id - created_on_time_str = cloud_task_run.created_on.isoformat() self.assertEqual( function_id, feconf.FUNCTION_ID_TO_FUNCTION_NAME_FOR_DEFERRED_JOBS[ @@ -924,10 +968,7 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: # via a deferred job. voiceover_services.regenerate_voiceovers_on_exploration_update( self.exploration_id, - updated_exp.title, updated_exp.version, - feconf.SYSTEM_COMMITTER_ID, - created_on_time_str, task_run_id, ) @@ -955,15 +996,16 @@ def test_regenerate_voiceovers_on_exploration_update(self) -> None: automated_voiceovers_audio_offsets_msecs = { 'content_0': dummy_audio_offset, 'content_3': dummy_audio_offset, + 'default_outcome_1': dummy_audio_offset, + 'ca_placeholder_2': dummy_audio_offset, } self.assertDictEqual( entity_voiceovers[0].automated_voiceovers_audio_offsets_msecs, automated_voiceovers_audio_offsets_msecs, ) - # The exploration contains two non-empty contents, content_0 and - # content_3, that are voiceovered automatically. + self.assertListEqual( - ['content_0', 'content_3'], + ['content_0', 'default_outcome_1', 'ca_placeholder_2', 'content_3'], list(entity_voiceovers[0].voiceovers_mapping.keys()), ) @@ -1015,7 +1057,6 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() function_id = cloud_task_runs[0].function_id task_run_id = cloud_task_runs[0].task_run_id - created_on_time_str = cloud_task_runs[0].created_on.isoformat() # Verifying that a Cloud Task run is created to regenerate the # voiceovers. @@ -1040,11 +1081,23 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: # via a deferred job. voiceover_services.regenerate_voiceovers_on_exploration_added_to_topic( self.exploration_id, - created_on_time_str, - feconf.SYSTEM_COMMITTER_ID, task_run_id, ) + updated_cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() + child_task_run_ids = [] + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + child_task_run_ids.append(cloud_run.task_run_id) + + for child_task_run_id in child_task_run_ids: + voiceover_services.regenerate_voiceovers_for_batch_contents( + self.exploration_id, task_run_id, child_task_run_id + ) + entity_voiceovers = ( voiceover_services.get_entity_voiceovers_for_given_exploration( self.exploration_id, 'exploration', 2 @@ -1102,10 +1155,9 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: # Simulating the frontend request that triggers voiceover regeneration # after an exploration update via a deferred job. - handler_url = '/regenerate_voiceover_on_exp_update/%s/%s/%s' % ( + handler_url = '/regenerate_voiceover_on_exp_update/%s/%s' % ( self.exploration_id, updated_exp.version, - updated_exp.title, ) csrf_token = self.get_new_csrf_token() self.post_json(handler_url, {}, csrf_token=csrf_token) @@ -1118,10 +1170,9 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: # Updating a curated exploration triggers voiceover regeneration via # the Cloud Task service, confirming that a deferred request exists in # the model. - cloud_task_run = cloud_task_runs[1] + cloud_task_run = cloud_task_runs[2] function_id = cloud_task_run.function_id task_run_id = cloud_task_run.task_run_id - created_on_time_str = cloud_task_run.created_on.isoformat() self.assertEqual( function_id, feconf.FUNCTION_ID_TO_FUNCTION_NAME_FOR_DEFERRED_JOBS[ @@ -1133,13 +1184,36 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: # via a deferred job. voiceover_services.regenerate_voiceovers_on_exploration_update( self.exploration_id, - updated_exp.title, updated_exp.version, - feconf.SYSTEM_COMMITTER_ID, - created_on_time_str, task_run_id, ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + second_iter_child_task_run_ids = [] + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + and cloud_run.task_run_id not in child_task_run_ids + ): + second_iter_child_task_run_ids.append(cloud_run.task_run_id) + + if ( + cloud_run.function_id + == 'regenerate_voiceovers_on_exploration_update' + ): + second_iter_parent_task_run_id = cloud_run.task_run_id + + for child_task_run_id in second_iter_child_task_run_ids: + voiceover_services.regenerate_voiceovers_for_batch_contents( + self.exploration_id, + second_iter_parent_task_run_id, + child_task_run_id, + ) + entity_voiceovers = sorted( voiceover_services.get_entity_voiceovers_for_given_exploration( self.exploration_id, 'exploration', 3 @@ -1161,7 +1235,7 @@ def test_regenerate_voiceovers_on_translation_addition(self) -> None: self.assertEqual(hindi_entity_voiceover.language_accent_code, 'hi-IN') self.assertListEqual( - ['content_0', 'content_3'], + ['content_0', 'default_outcome_1', 'ca_placeholder_2', 'content_3'], list(english_entity_voiceover.voiceovers_mapping.keys()), ) # Hindi translation was added only for the first content. @@ -1255,6 +1329,30 @@ def test_regenerate_voiceovers_on_translation_suggestion_acceptance( }, csrf_token=csrf_token_2, ) + cloud_task_runs = taskqueue_services.get_all_cloud_task_runs() + + voiceover_services.regenerate_voiceovers_after_accepting_suggestion( + suggestion_to_accept['suggestion_id'], + cloud_task_runs[1].task_run_id, + ) + + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + child_task_run_ids = [] + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + child_task_run_ids.append(cloud_run.task_run_id) + for child_task_run_id in child_task_run_ids: + voiceover_services.regenerate_voiceovers_for_batch_contents( + self.exploration_id, + cloud_task_runs[1].task_run_id, + child_task_run_id, + ) self.logout() @@ -1622,11 +1720,9 @@ class RegenerateVoiceoversForExplorationHandlerTests( def mock_defer( self, _function_id: str, - _queue_id: str, + _queue_name: str, _exploration_id: str, _language_accent_code: str, - _user_id: str, - _datetime_str: str, ) -> None: pass diff --git a/core/domain/beam_job_services.py b/core/domain/beam_job_services.py index 7ffd135a8643d..c81eeee3cc1aa 100644 --- a/core/domain/beam_job_services.py +++ b/core/domain/beam_job_services.py @@ -24,7 +24,7 @@ from core.jobs import registry as jobs_registry from core.platform import models -from typing import List, Optional, Type +from typing import Dict, List, Optional, Type MYPY = False if MYPY: # pragma: no cover @@ -38,6 +38,7 @@ def run_beam_job( job_name: Optional[str] = None, job_class: Optional[Type[base_jobs.JobBase]] = None, + parameterized_args: Optional[Dict[str, str]] = None, ) -> beam_job_domain.BeamJobRun: """Starts a new Apache Beam job and returns metadata about its execution. @@ -46,6 +47,10 @@ def run_beam_job( job_class must not be None. job_class: type(JobBase). A subclass of JobBase to begin running. This value takes precedence over job_name. + parameterized_args: dict(str, str). The arguments to pass to the + job when it is run. The keys of the dict should be the names of the + arguments as defined in the job's class definition, and the values + should be the corresponding values to use for those arguments. Returns: BeamJobRun. Metadata about the run's execution. @@ -60,7 +65,10 @@ def run_beam_job( raise ValueError('Must specify the job class or name to run') run_synchronously = constants.EMULATOR_MODE - run_model = jobs_manager.run_job(job_class, run_synchronously) + + run_model = jobs_manager.run_job( + job_class, run_synchronously, parameterized_args=parameterized_args + ) return get_beam_job_run_from_model(run_model) diff --git a/core/domain/blog_services.py b/core/domain/blog_services.py index 5662fa36d591e..777defaf3c5a7 100644 --- a/core/domain/blog_services.py +++ b/core/domain/blog_services.py @@ -412,6 +412,26 @@ def get_blog_post_rights( return get_blog_post_rights_from_model(model) +def get_total_number_of_matching_blog_posts( + query_string: str, tags: List[str] +) -> int: + """Returns the total number of blog posts matching the search query and tags.""" + valid_blog_post_ids: List[str] = [] + search_offset: Optional[int] = None + + for _ in range(MAX_ITERATIONS): + remaining_to_fetch = 1000 + + batch_ids, search_offset = get_blog_post_ids_matching_query( + query_string, tags, size=remaining_to_fetch, offset=search_offset + ) + valid_blog_post_ids.extend(batch_ids) + + if search_offset is None: + break + return len(valid_blog_post_ids) + + def get_published_blog_post_summaries_by_user_id( user_id: str, max_limit: int, offset: int = 0 ) -> List[blog_domain.BlogPostSummary]: diff --git a/core/domain/cloud_task_domain.py b/core/domain/cloud_task_domain.py index 66eaded5d2d36..db037ce12fb7a 100644 --- a/core/domain/cloud_task_domain.py +++ b/core/domain/cloud_task_domain.py @@ -91,6 +91,34 @@ def to_dict(self) -> CloudTaskRunDict: 'created_on': self.created_on.isoformat(), } + def to_dict_with_timezone_info(self) -> CloudTaskRunDict: + """Returns a dictionary representation of this domain object with timezone + information included in the datetime fields. + + Returns: + CloudTaskRunDict. A dictionary representation of the CloudTaskRun + object, with keys matching the attributes of the object and timezone + information included in the datetime fields. + """ + return { + 'task_run_id': self.task_run_id, + 'cloud_task_name': self.cloud_task_name, + 'task_id': self.task_id, + 'queue_id': self.queue_id, + 'latest_job_state': self.latest_job_state, + 'function_id': self.function_id, + 'exception_messages_for_failed_runs': ( + self.exception_messages_for_failed_runs + ), + 'current_retry_attempt': self.current_retry_attempt, + 'last_updated': self.last_updated.replace( + tzinfo=datetime.timezone.utc + ).isoformat(), + 'created_on': self.created_on.replace( + tzinfo=datetime.timezone.utc + ).isoformat(), + } + @classmethod def from_dict(cls, cloud_task_run_dict: CloudTaskRunDict) -> CloudTaskRun: """Returns a domain object from a dictionary. @@ -123,15 +151,15 @@ def from_dict(cls, cloud_task_run_dict: CloudTaskRunDict) -> CloudTaskRun: ) -class VoiceoverRegenerationTaskMappingDict(TypedDict): - """Dictionary representing the VoiceoverRegenerationTaskMapping object.""" +class VoiceoverRegenerationJobDict(TypedDict): + """Dictionary representing the VoiceoverRegenerationJob object.""" exploration_id: str task_run_id: str language_accent_to_content_status_map: Dict[str, Dict[str, str]] -class VoiceoverRegenerationTaskMapping: +class VoiceoverRegenerationJob: """Domain object class that models the voiceover regeneration request for an exploration, associated with a specific cloud task run. """ @@ -142,7 +170,7 @@ def __init__( task_run_id: str, language_accent_to_content_status_map: Dict[str, Dict[str, str]], ) -> None: - """Initializes a VoiceoverRegenerationTaskMapping domain object. + """Initializes a VoiceoverRegenerationJob domain object. Args: exploration_id: str. The ID of the exploration. @@ -156,12 +184,12 @@ def __init__( language_accent_to_content_status_map ) - def to_dict(self) -> VoiceoverRegenerationTaskMappingDict: + def to_dict(self) -> VoiceoverRegenerationJobDict: """Returns a dictionary representation of this domain object. Returns: dict. A dictionary representation of the - VoiceoverRegenerationTaskMapping object, with keys matching the + VoiceoverRegenerationJob object, with keys matching the attributes of the object. """ @@ -176,44 +204,42 @@ def to_dict(self) -> VoiceoverRegenerationTaskMappingDict: @classmethod def from_dict( cls, - voiceover_regeneration_task_mapping_dict: VoiceoverRegenerationTaskMappingDict, - ) -> VoiceoverRegenerationTaskMapping: - """Returns an instance of VoiceoverRegenerationTaskMapping from the + voiceover_regeneration_job_dict: VoiceoverRegenerationJobDict, + ) -> VoiceoverRegenerationJob: + """Returns an instance of VoiceoverRegenerationJob from the given dictionary. Args: - voiceover_regeneration_task_mapping_dict: dict. A dictionary - representation of the VoiceoverRegenerationTaskMapping object. + voiceover_regeneration_job_dict: dict. A dictionary + representation of the VoiceoverRegenerationJob object. Returns: - VoiceoverRegenerationTaskMapping. A VoiceoverRegenerationTaskMapping + VoiceoverRegenerationJob. A VoiceoverRegenerationJob domain object created from the given dict representation. """ return cls( - exploration_id=voiceover_regeneration_task_mapping_dict[ - 'exploration_id' - ], - task_run_id=voiceover_regeneration_task_mapping_dict['task_run_id'], + exploration_id=voiceover_regeneration_job_dict['exploration_id'], + task_run_id=voiceover_regeneration_job_dict['task_run_id'], language_accent_to_content_status_map=( - voiceover_regeneration_task_mapping_dict[ + voiceover_regeneration_job_dict[ 'language_accent_to_content_status_map' ] ), ) @classmethod - def create_default_voiceover_regeneration_task_mapping( + def create_default( cls, exploration_id: str, task_run_id: str - ) -> VoiceoverRegenerationTaskMapping: - """Creates a default voiceover regeneration task mapping. + ) -> VoiceoverRegenerationJob: + """Creates a default voiceover regeneration job instance. Args: exploration_id: str. The ID of the exploration. task_run_id: str. The ID of the cloud task run. Returns: - VoiceoverRegenerationTaskMapping. The created voiceover - regeneration task mapping. + VoiceoverRegenerationJob. The created voiceover regeneration job + instance with an empty language accent to content status map. """ return cls( exploration_id=exploration_id, @@ -241,7 +267,86 @@ def are_all_voiceovers_generated(self) -> bool: return False return True - def update_final_content_status_for_cloud_task_run( + def are_all_voiceovers_attempted(self) -> bool: + """Checks if all the contents for the voiceover regeneration request + have been attempted i.e., either succeeded or failed, none + of them are still generating. + + Returns: + bool. Whether all contents have been attempted or not. + """ + for ( + content_id_to_regeneration_status + ) in self.language_accent_to_content_status_map.values(): + for ( + regeneration_status + ) in content_id_to_regeneration_status.values(): + if ( + regeneration_status + == feconf.VoiceoverRegenerationState.GENERATING.value + ): + return False + return True + + def update_failed_content_status( + self, language_accent_code: str, failed_content_ids: List[str] + ) -> None: + """Updates the content-status map for a given language-accent code by + marking the content IDs in failed_content_ids as FAILED. + + Args: + language_accent_code: str. The language accent code. + failed_content_ids: List[str]. The list of content IDs for which + voiceover regeneration has failed. + """ + content_status_map = self.language_accent_to_content_status_map.get( + language_accent_code, {} + ) + + for content_id in failed_content_ids: + if content_id in content_status_map: + content_status_map[content_id] = ( + feconf.VoiceoverRegenerationState.FAILED.value + ) + + def update_succeeded_content_status( + self, language_accent_code: str, succeeded_content_ids: List[str] + ) -> None: + """Updates the content-status map for a given language-accent code by + marking the content IDs in succeeded_content_ids as SUCCEEDED. + + Args: + language_accent_code: str. The language accent code. + succeeded_content_ids: List[str]. The list of content IDs for which + voiceover regeneration has succeeded. + """ + content_status_map = self.language_accent_to_content_status_map.get( + language_accent_code, {} + ) + + for content_id in succeeded_content_ids: + if content_id in content_status_map: + content_status_map[content_id] = ( + feconf.VoiceoverRegenerationState.SUCCEEDED.value + ) + + def update_remaining_content_status_as_succeeded(self) -> None: + """Updates the content-status map for a given language-accent code by + marking all content IDs which are still GENERATING as SUCCEEDED. + """ + for ( + content_status_map + ) in self.language_accent_to_content_status_map.values(): + for content_id, regeneration_status in content_status_map.items(): + if ( + regeneration_status + == feconf.VoiceoverRegenerationState.GENERATING.value + ): + content_status_map[content_id] = ( + feconf.VoiceoverRegenerationState.SUCCEEDED.value + ) + + def update_final_content_status( self, language_accent_code: str, failed_content_ids: List[str] ) -> None: """Updates the content-status map for a given language-accent code by @@ -287,3 +392,121 @@ def add_language_accent_to_content_status_map( self.language_accent_to_content_status_map[language_accent_code] = ( content_status_map ) + + def count_total_failed_contents(self) -> int: + """Counts the total number of contents for which voiceover regeneration + has failed. + + Returns: + int. The total number of contents for which voiceover regeneration + has failed. + """ + total_failed_contents = 0 + for ( + content_id_to_regeneration_status + ) in self.language_accent_to_content_status_map.values(): + for ( + regeneration_status + ) in content_id_to_regeneration_status.values(): + if ( + regeneration_status + == feconf.VoiceoverRegenerationState.FAILED.value + ): + total_failed_contents += 1 + + return total_failed_contents + + +class VoiceoverRegenerationTaskBatchDict(TypedDict): + """Dictionary representing the VoiceoverRegenerationTaskBatch object.""" + + parent_cloud_task_run_id: str + child_cloud_task_run_id: str + exploration_id: str + exploration_version: int + language_accent_code: str + content_ids_to_contents_map: Dict[str, str] + + +class VoiceoverRegenerationTaskBatch: + """Voiceover regeneration for a large number of contents within a single + Cloud Task run (deferred request) significantly increases the workload and + may lead to timeout failures due to Gunicorn limitations. + + To mitigate this issue, a single deferred regeneration task is split into + multiple smaller batches, organized in a parent-child relationship between + Cloud Task runs. + + This class is the domain class representation for + VoiceoverRegenerationBatchExecutionModel. + """ + + def __init__( + self, + parent_cloud_task_run_id: str, + child_cloud_task_run_id: str, + exploration_id: str, + exploration_version: int, + language_accent_code: str, + content_ids_to_contents_map: Dict[str, str], + ) -> None: + self.parent_cloud_task_run_id = parent_cloud_task_run_id + self.child_cloud_task_run_id = child_cloud_task_run_id + self.exploration_id = exploration_id + self.exploration_version = exploration_version + self.language_accent_code = language_accent_code + self.content_ids_to_contents_map = content_ids_to_contents_map + + def to_dict(self) -> VoiceoverRegenerationTaskBatchDict: + """Returns a dictionary representation of this domain object. + + Returns: + dict. A dictionary representation of the + VoiceoverRegenerationTaskBatch object, with keys matching the + attributes of the object. + """ + return { + 'parent_cloud_task_run_id': self.parent_cloud_task_run_id, + 'child_cloud_task_run_id': self.child_cloud_task_run_id, + 'exploration_id': self.exploration_id, + 'exploration_version': self.exploration_version, + 'language_accent_code': self.language_accent_code, + 'content_ids_to_contents_map': self.content_ids_to_contents_map, + } + + @classmethod + def from_dict( + cls, + voiceover_regeneration_task_batch_dict: VoiceoverRegenerationTaskBatchDict, + ) -> VoiceoverRegenerationTaskBatch: + """Returns an instance of VoiceoverRegenerationTaskBatch from the + given dictionary. + + Args: + voiceover_regeneration_task_batch_dict: dict. A dictionary + representation of the VoiceoverRegenerationTaskBatch object. + + Returns: + VoiceoverRegenerationTaskBatch. A VoiceoverRegenerationTaskBatch + domain object created from the given dict representation. + """ + return cls( + parent_cloud_task_run_id=voiceover_regeneration_task_batch_dict[ + 'parent_cloud_task_run_id' + ], + child_cloud_task_run_id=voiceover_regeneration_task_batch_dict[ + 'child_cloud_task_run_id' + ], + exploration_id=voiceover_regeneration_task_batch_dict[ + 'exploration_id' + ], + exploration_version=voiceover_regeneration_task_batch_dict[ + 'exploration_version' + ], + language_accent_code=voiceover_regeneration_task_batch_dict[ + 'language_accent_code' + ], + content_ids_to_contents_map=voiceover_regeneration_task_batch_dict[ + 'content_ids_to_contents_map' + ], + ) diff --git a/core/domain/cloud_task_domain_test.py b/core/domain/cloud_task_domain_test.py index 5ba4393fb96fe..b28fc30fb4529 100644 --- a/core/domain/cloud_task_domain_test.py +++ b/core/domain/cloud_task_domain_test.py @@ -107,9 +107,33 @@ def test_should_create_domain_object_from_dict(self) -> None: self.assertEqual(cloud_task_run.to_dict(), cloud_task_run_dict) + def test_should_convert_datetime_fields_with_timezone_info(self) -> None: + cloud_task_run = cloud_task_domain.CloudTaskRun( + task_run_id='cloud_task_run_id', + cloud_task_name='projects/dev-project-id/locations/us-central/' + 'queues/test_queue_name/tasks/task_id', + task_id='task_id', + queue_id='test_queue_name', + latest_job_state='running', + function_id='delete_exps_from_user_models', + exception_messages_for_failed_runs=[], + current_retry_attempt=0, + last_updated=datetime.datetime(2026, 1, 2, 3, 4, 5), + created_on=datetime.datetime(2026, 1, 2, 3, 4, 6), + ) + + cloud_task_run_dict = cloud_task_run.to_dict_with_timezone_info() + + self.assertEqual( + cloud_task_run_dict['last_updated'], '2026-01-02T03:04:05+00:00' + ) + self.assertEqual( + cloud_task_run_dict['created_on'], '2026-01-02T03:04:06+00:00' + ) -class VoiceoverRegenerationTaskMappingTests(test_utils.GenericTestBase): - """Unit tests for VoiceoverRegenerationTaskMapping domain object.""" + +class VoiceoverRegenerationJobTests(test_utils.GenericTestBase): + """Unit tests for VoiceoverRegenerationJob domain object.""" def test_should_create_domain_object_correctly(self) -> None: exploration_id = 'exp_id' @@ -122,7 +146,7 @@ def test_should_create_domain_object_correctly(self) -> None: } voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping( + cloud_task_domain.VoiceoverRegenerationJob( exploration_id, task_run_id, language_accent_to_content_status_map, @@ -151,7 +175,7 @@ def test_should_create_domain_object_from_dict(self) -> None: } voiceover_regeneration_task_mapping_dict: ( - cloud_task_domain.VoiceoverRegenerationTaskMappingDict + cloud_task_domain.VoiceoverRegenerationJobDict ) = { 'exploration_id': exploration_id, 'task_run_id': task_run_id, @@ -161,7 +185,7 @@ def test_should_create_domain_object_from_dict(self) -> None: } voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping.from_dict( + cloud_task_domain.VoiceoverRegenerationJob.from_dict( voiceover_regeneration_task_mapping_dict ) ) @@ -175,8 +199,10 @@ def test_should_be_able_to_create_default_object(self) -> None: exploration_id = 'exp_id' task_run_id = 'task_run_id' - voiceover_regeneration_task_mapping = cloud_task_domain.VoiceoverRegenerationTaskMapping.create_default_voiceover_regeneration_task_mapping( - exploration_id, task_run_id + voiceover_regeneration_task_mapping = ( + cloud_task_domain.VoiceoverRegenerationJob.create_default( + exploration_id, task_run_id + ) ) self.assertEqual( @@ -205,7 +231,7 @@ def test_should_verify_if_all_voiceovers_are_generated(self) -> None: } voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping( + cloud_task_domain.VoiceoverRegenerationJob( exploration_id, task_run_id, language_accent_to_content_status_map, @@ -227,7 +253,7 @@ def test_should_verify_if_all_voiceovers_are_generated(self) -> None: voiceover_regeneration_task_mapping.are_all_voiceovers_generated() ) - def test_should_update_final_content_status_for_cloud_task_run( + def test_should_update_final_content_status_successfully( self, ) -> None: exploration_id = 'exp_id' @@ -241,14 +267,17 @@ def test_should_update_final_content_status_for_cloud_task_run( } voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping( + cloud_task_domain.VoiceoverRegenerationJob( exploration_id, task_run_id, language_accent_to_content_status_map, ) ) + self.assertFalse( + voiceover_regeneration_task_mapping.are_all_voiceovers_attempted() + ) - voiceover_regeneration_task_mapping.update_final_content_status_for_cloud_task_run( + voiceover_regeneration_task_mapping.update_final_content_status( 'en-US', ['content_1'] ) @@ -264,6 +293,12 @@ def test_should_update_final_content_status_for_cloud_task_run( voiceover_regeneration_task_mapping.language_accent_to_content_status_map, expected_language_accent_to_content_status_map, ) + self.assertTrue( + voiceover_regeneration_task_mapping.are_all_voiceovers_attempted() + ) + self.assertEqual( + voiceover_regeneration_task_mapping.count_total_failed_contents(), 1 + ) def test_should_add_language_accent_to_content_status_map(self) -> None: exploration_id = 'exp_id' @@ -271,7 +306,7 @@ def test_should_add_language_accent_to_content_status_map(self) -> None: language_accent_to_content_status_map: Dict[str, Dict[str, str]] = {} voiceover_regeneration_task_mapping = ( - cloud_task_domain.VoiceoverRegenerationTaskMapping( + cloud_task_domain.VoiceoverRegenerationJob( exploration_id, task_run_id, language_accent_to_content_status_map, @@ -293,3 +328,229 @@ def test_should_add_language_accent_to_content_status_map(self) -> None: voiceover_regeneration_task_mapping.language_accent_to_content_status_map, expected_language_accent_to_content_status_map, ) + + def test_should_successfully_update_status_of_contents(self) -> None: + exploration_id = 'exp_id' + task_run_id = 'task_run_id' + language_accent_to_content_status_map = { + 'en-US': { + 'content_0': 'GENERATING', + 'content_1': 'GENERATING', + } + } + + voiceover_regeneration_task_mapping = ( + cloud_task_domain.VoiceoverRegenerationJob( + exploration_id, + task_run_id, + language_accent_to_content_status_map, + ) + ) + + voiceover_regeneration_task_mapping.update_succeeded_content_status( + 'en-US', ['content_0'] + ) + voiceover_regeneration_task_mapping.update_failed_content_status( + 'en-US', ['content_1'] + ) + + expected_language_accent_to_content_status_map = { + 'en-US': { + 'content_0': 'SUCCEEDED', + 'content_1': 'FAILED', + } + } + + self.assertEqual( + voiceover_regeneration_task_mapping.language_accent_to_content_status_map, + expected_language_accent_to_content_status_map, + ) + + def test_should_update_remaining_content_status_as_succeeded(self) -> None: + exploration_id = 'exp_id' + task_run_id = 'task_run_id' + language_accent_to_content_status_map = { + 'en-US': { + 'content_0': 'GENERATING', + 'content_1': 'FAILED', + 'content_2': 'SUCCEEDED', + }, + 'hi-IN': { + 'content_3': 'GENERATING', + }, + } + + voiceover_regeneration_task_mapping = ( + cloud_task_domain.VoiceoverRegenerationJob( + exploration_id, + task_run_id, + language_accent_to_content_status_map, + ) + ) + + self.assertFalse( + voiceover_regeneration_task_mapping.are_all_voiceovers_attempted() + ) + + ( + voiceover_regeneration_task_mapping.update_remaining_content_status_as_succeeded() + ) + + expected_language_accent_to_content_status_map = { + 'en-US': { + 'content_0': 'SUCCEEDED', + 'content_1': 'FAILED', + 'content_2': 'SUCCEEDED', + }, + 'hi-IN': { + 'content_3': 'SUCCEEDED', + }, + } + + self.assertEqual( + voiceover_regeneration_task_mapping.language_accent_to_content_status_map, + expected_language_accent_to_content_status_map, + ) + self.assertTrue( + voiceover_regeneration_task_mapping.are_all_voiceovers_attempted() + ) + + +class VoiceoverRegenerationTaskBatchTests(test_utils.GenericTestBase): + """Unit tests for VoiceoverRegenerationTaskBatch domain object.""" + + def test_should_create_domain_object_correctly(self) -> None: + parent_cloud_task_run_id = 'parent_task_run_id' + child_cloud_task_run_id = 'child_task_run_id_1' + exploration_id = 'exp_id' + exploration_version = 1 + language_accent_code = 'en-US' + content_ids_to_contents_map = { + 'content_0': 'This is content 0', + 'content_1': 'This is content 1', + } + + voiceover_regeneration_task_batch = ( + cloud_task_domain.VoiceoverRegenerationTaskBatch( + parent_cloud_task_run_id, + child_cloud_task_run_id, + exploration_id, + exploration_version, + language_accent_code, + content_ids_to_contents_map, + ) + ) + + self.assertEqual( + voiceover_regeneration_task_batch.parent_cloud_task_run_id, + parent_cloud_task_run_id, + ) + self.assertEqual( + voiceover_regeneration_task_batch.child_cloud_task_run_id, + child_cloud_task_run_id, + ) + self.assertEqual( + voiceover_regeneration_task_batch.exploration_id, exploration_id + ) + self.assertEqual( + voiceover_regeneration_task_batch.exploration_version, + exploration_version, + ) + self.assertEqual( + voiceover_regeneration_task_batch.language_accent_code, + language_accent_code, + ) + self.assertEqual( + voiceover_regeneration_task_batch.content_ids_to_contents_map, + content_ids_to_contents_map, + ) + + def test_should_create_domain_object_from_dict(self) -> None: + parent_cloud_task_run_id = 'parent_task_run_id' + child_cloud_task_run_id = 'child_task_run_id_1' + exploration_id = 'exp_id' + exploration_version = 1 + language_accent_code = 'en-US' + content_ids_to_contents_map = { + 'content_0': 'This is content 0', + 'content_1': 'This is content 1', + } + + voiceover_regeneration_task_batch_dict: ( + cloud_task_domain.VoiceoverRegenerationTaskBatchDict + ) = { + 'parent_cloud_task_run_id': parent_cloud_task_run_id, + 'child_cloud_task_run_id': child_cloud_task_run_id, + 'exploration_id': exploration_id, + 'exploration_version': exploration_version, + 'language_accent_code': language_accent_code, + 'content_ids_to_contents_map': content_ids_to_contents_map, + } + + voiceover_regeneration_task_batch = ( + cloud_task_domain.VoiceoverRegenerationTaskBatch.from_dict( + voiceover_regeneration_task_batch_dict + ) + ) + + self.assertEqual( + voiceover_regeneration_task_batch.to_dict(), + voiceover_regeneration_task_batch_dict, + ) + + def test_should_convert_to_dict_correctly(self) -> None: + parent_cloud_task_run_id = 'parent_task_run_id' + child_cloud_task_run_id = 'child_task_run_id_1' + exploration_id = 'exp_id' + exploration_version = 2 + language_accent_code = 'hi-IN' + content_ids_to_contents_map = { + 'content_0': 'Content 0 text', + } + + voiceover_regeneration_task_batch = ( + cloud_task_domain.VoiceoverRegenerationTaskBatch( + parent_cloud_task_run_id, + child_cloud_task_run_id, + exploration_id, + exploration_version, + language_accent_code, + content_ids_to_contents_map, + ) + ) + + expected_dict = { + 'parent_cloud_task_run_id': parent_cloud_task_run_id, + 'child_cloud_task_run_id': child_cloud_task_run_id, + 'exploration_id': exploration_id, + 'exploration_version': exploration_version, + 'language_accent_code': language_accent_code, + 'content_ids_to_contents_map': content_ids_to_contents_map, + } + + self.assertEqual( + voiceover_regeneration_task_batch.to_dict(), expected_dict + ) + + def test_should_handle_empty_content_map(self) -> None: + parent_cloud_task_run_id = 'parent_task_run_id' + child_cloud_task_run_id = 'child_task_run_id_1' + exploration_id = 'exp_id' + exploration_version = 1 + language_accent_code = 'en-US' + content_ids_to_contents_map: Dict[str, str] = {} + + voiceover_regeneration_task_batch = ( + cloud_task_domain.VoiceoverRegenerationTaskBatch( + parent_cloud_task_run_id, + child_cloud_task_run_id, + exploration_id, + exploration_version, + language_accent_code, + content_ids_to_contents_map, + ) + ) + + self.assertEqual( + voiceover_regeneration_task_batch.content_ids_to_contents_map, {} + ) diff --git a/core/domain/email_manager.py b/core/domain/email_manager.py index 42ca0297901e8..519829feecddf 100644 --- a/core/domain/email_manager.py +++ b/core/domain/email_manager.py @@ -36,6 +36,7 @@ story_domain, subscription_services, suggestion_registry, + taskqueue_services, user_services, ) from core.platform import models @@ -672,16 +673,36 @@ def _send_email_transactional() -> None: """Sends the email to a single recipient.""" sender_name_email = '%s <%s>' % (sender_name, sender_email) - email_services.send_mail( - sender_name_email, - recipient_email_address, - email_subject, - cleaned_plaintext_body, - cleaned_html_body, - cc_emails=cc_emails, - bcc_admin=bcc_admin, - attachments=attachments, - ) + try: + email_services.send_mail( + sender_name_email, + recipient_email_address, + email_subject, + cleaned_plaintext_body, + cleaned_html_body, + cc_emails=cc_emails, + bcc_admin=bcc_admin, + attachments=attachments, + ) + except Exception as e: + logging.error( + 'Email to %s failed to send: %s. Enqueuing for retry.', + recipient_email_address, + e, + ) + + payload = { + 'sender_email': sender_name_email, + 'recipient_id': recipient_email_address, + 'subject': email_subject, + 'html_body': cleaned_html_body, + 'text_body': cleaned_plaintext_body, + } + + taskqueue_services.enqueue_task( + feconf.TASK_URL_RETRY_FAILED_EMAIL, payload, 0 + ) + email_models.SentEmailModel.create( recipient_id, recipient_email_address, @@ -756,14 +777,31 @@ def _send_bulk_mail_transactional(instance_id: str) -> None: """ sender_name_email = '%s <%s>' % (sender_name, sender_email) - email_services.send_bulk_mail( - sender_name_email, - recipient_emails, - email_subject, - cleaned_plaintext_body, - cleaned_html_body, - attachments, - ) + try: + email_services.send_bulk_mail( + sender_name_email, + recipient_emails, + email_subject, + cleaned_plaintext_body, + cleaned_html_body, + attachments, + ) + except Exception as e: + logging.error( + 'Bulk email failed to send: %s. Enqueuing for retry.', e + ) + + for recipient_email in recipient_emails: + payload = { + 'sender_email': sender_name_email, + 'recipient_id': recipient_email, + 'subject': email_subject, + 'html_body': cleaned_html_body, + 'text_body': cleaned_plaintext_body, + } + taskqueue_services.enqueue_task( + feconf.TASK_URL_RETRY_FAILED_EMAIL, payload, 0 + ) email_models.BulkEmailModel.create( instance_id, diff --git a/core/domain/email_manager_test.py b/core/domain/email_manager_test.py index d626cadc14f77..b39ea688dd94b 100644 --- a/core/domain/email_manager_test.py +++ b/core/domain/email_manager_test.py @@ -24,6 +24,7 @@ from core.constants import constants from core.domain import ( email_manager, + email_services, exp_domain, html_cleaner, platform_parameter_domain, @@ -37,6 +38,7 @@ subscription_services, suggestion_registry, suggestion_services, + taskqueue_services, translation_domain, user_services, ) @@ -9330,3 +9332,82 @@ def test_sends_email_to_tech_leads_on_regeneration_failure(self) -> None: sent_email_model.subject, '[Attention needed] Automatic Voiceover Generation Failed', ) + + +class EmailRetryQueueTests(test_utils.EmailTestBase): + """Tests the retry logic when email sending fails.""" + + USER_A_EMAIL = 'a@example.com' + + def setUp(self) -> None: + super().setUp() + self.signup(self.USER_A_EMAIL, 'userA') + self.user_a_id = self.get_user_id_from_email(self.USER_A_EMAIL) + + def test_failed_send_mail_enqueues_retry_task(self) -> None: + def mock_send_mail(*_args: str, **_kwargs: str) -> None: + raise Exception('Simulated email failure') + + enqueued_tasks = [] + + def mock_enqueue_task( + url: str, payload: dict[str, str], _delay: int + ) -> None: + enqueued_tasks.append((url, payload)) + + send_mail_swap = self.swap(email_services, 'send_mail', mock_send_mail) + enqueue_task_swap = self.swap( + taskqueue_services, 'enqueue_task', mock_enqueue_task + ) + + with send_mail_swap, enqueue_task_swap: + email_manager._send_email( # pylint: disable=protected-access + self.user_a_id, + feconf.SYSTEM_COMMITTER_ID, + feconf.EMAIL_INTENT_SIGNUP, + 'Subject', + 'Body', + 'sender@example.com', + ) + + self.assertEqual(len(enqueued_tasks), 1) + self.assertEqual( + enqueued_tasks[0][0], feconf.TASK_URL_RETRY_FAILED_EMAIL + ) + self.assertEqual(enqueued_tasks[0][1]['subject'], 'Subject') + + def test_failed_send_bulk_mail_enqueues_retry_task(self) -> None: + def mock_send_bulk_mail(*_args: str, **_kwargs: str) -> None: + raise Exception('Simulated bulk email failure') + + enqueued_tasks = [] + + def mock_enqueue_task( + url: str, payload: dict[str, str], _delay: int + ) -> None: + enqueued_tasks.append((url, payload)) + + send_bulk_mail_swap = self.swap( + email_services, 'send_bulk_mail', mock_send_bulk_mail + ) + enqueue_task_swap = self.swap( + taskqueue_services, 'enqueue_task', mock_enqueue_task + ) + + with send_bulk_mail_swap, enqueue_task_swap: + email_manager._send_bulk_mail( # pylint: disable=protected-access + [self.user_a_id], + feconf.SYSTEM_COMMITTER_ID, + feconf.BULK_EMAIL_INTENT_MARKETING, + 'Bulk Subject', + 'Bulk Body', + 'sender@example.com', + 'Sender Name', + 'instance_id', + ) + + self.assertEqual(len(enqueued_tasks), 1) + self.assertEqual( + enqueued_tasks[0][0], feconf.TASK_URL_RETRY_FAILED_EMAIL + ) + self.assertEqual(enqueued_tasks[0][1]['subject'], 'Bulk Subject') diff --git a/core/domain/exp_services.py b/core/domain/exp_services.py index 1cb73af057984..d97901db91b34 100644 --- a/core/domain/exp_services.py +++ b/core/domain/exp_services.py @@ -476,6 +476,178 @@ def export_states_to_yaml( return exploration_dict +def get_content_updates_from_cmd_edit_state_property_change( + change: exp_domain.ExplorationChange, +) -> Dict[str, str]: + """Extracts content ids and content values from CMD_EDIT_STATE_PROPERTY. + + Args: + change: ExplorationChange. The exploration change object. + + Returns: + dict(str, str). A mapping from content_id to content html. + """ + content_id_to_content_value: Dict[str, str] = {} + + if change.cmd != exp_domain.CMD_EDIT_STATE_PROPERTY: + return content_id_to_content_value + + if change.new_value is None: + return content_id_to_content_value + + def add_subtitled_html_from_dict( + subtitled_html: state_domain.SubtitledHtmlDict, + ) -> None: + """Adds a mapping from a subtitled html dict, if valid.""" + conten_id = subtitled_html.get('content_id') + + content_value = None + if subtitled_html.get('html'): + content_value = subtitled_html.get('html') + + if isinstance(conten_id, str) and isinstance(content_value, str): + content_id_to_content_value[conten_id] = content_value + + def add_subtitled_unicode_from_dict( + subtitled_unicode: state_domain.SubtitledUnicodeDict, + ) -> None: + """Adds a mapping from a subtitled unicode dict, if valid.""" + conten_id = subtitled_unicode.get('content_id') + + content_value = None + if subtitled_unicode.get('unicode_str'): + content_value = subtitled_unicode.get('unicode_str') + + if isinstance(conten_id, str) and isinstance(content_value, str): + content_id_to_content_value[conten_id] = content_value + + if change.property_name == exp_domain.STATE_PROPERTY_CONTENT: + # Here we use cast because this 'if' condition forces change to have + # type EditExpStatePropertyContentCmd. + edit_content_cmd = cast( + exp_domain.EditExpStatePropertyContentCmd, change + ) + add_subtitled_html_from_dict(edit_content_cmd.new_value) + elif ( + change.property_name + == exp_domain.STATE_PROPERTY_INTERACTION_DEFAULT_OUTCOME + ): + # Here we use cast because this 'elif' condition forces change to have + # type EditExpStatePropertyInteractionDefaultOutcomeCmd. + edit_interaction_default_outcome_cmd = cast( + exp_domain.EditExpStatePropertyInteractionDefaultOutcomeCmd, + change, + ) + + add_subtitled_html_from_dict( + edit_interaction_default_outcome_cmd.new_value['feedback'] + ) + elif ( + change.property_name + == exp_domain.STATE_PROPERTY_INTERACTION_ANSWER_GROUPS + ): + # Here we use cast because this 'elif' condition forces change to have + # type EditExpStatePropertyInteractionAnswerGroupsCmd. + edit_interaction_answer_group_cmd = cast( + exp_domain.EditExpStatePropertyInteractionAnswerGroupsCmd, + change, + ) + answer_group_dicts = edit_interaction_answer_group_cmd.new_value or [] + + for answer_group_dict in answer_group_dicts: + add_subtitled_html_from_dict( + answer_group_dict['outcome']['feedback'] + ) + elif change.property_name == exp_domain.STATE_PROPERTY_INTERACTION_HINTS: + # Here we use cast because this 'elif' condition forces change to have + # type EditExpStatePropertyInteractionHintsCmd. + edit_state_interaction_hints_cmd = cast( + exp_domain.EditExpStatePropertyInteractionHintsCmd, + change, + ) + hint_dicts = edit_state_interaction_hints_cmd.new_value or [] + + for hint_dict in hint_dicts: + add_subtitled_html_from_dict(hint_dict['hint_content']) + elif change.property_name == exp_domain.STATE_PROPERTY_INTERACTION_SOLUTION: + # Here we use cast because this 'elif' condition forces change to have + # type EditExpStatePropertyInteractionSolutionCmd. + edit_interaction_solution_cmd = cast( + exp_domain.EditExpStatePropertyInteractionSolutionCmd, + change, + ) + add_subtitled_html_from_dict( + edit_interaction_solution_cmd.new_value['explanation'] + ) + elif ( + change.property_name == exp_domain.STATE_PROPERTY_INTERACTION_CUST_ARGS + ): + # Here we use cast because this 'elif' condition forces change to have + # type EditExpStatePropertyInteractionCustArgsCmd. + edit_interaction_cust_arg_cmd = cast( + exp_domain.EditExpStatePropertyInteractionCustArgsCmd, + change, + ) + customization_arg_dicts = edit_interaction_cust_arg_cmd.new_value or {} + for cust_arg_dict in customization_arg_dicts.values(): + for cust_arg_value in cust_arg_dict.values(): + # Each of these conversions are intended to be run on every + # single item. + try: + # Here we use cast because we are narrowing down the type + # UnionOfCustomizationArgsDictValues dict to SubtitledHtmlDict. + cust_arg_subtitled_html_dict = cast( + state_domain.SubtitledHtmlDict, cust_arg_value + ) + add_subtitled_html_from_dict(cust_arg_subtitled_html_dict) + except Exception: + pass + + try: + # Here we use cast because we are narrowing down the type + # UnionOfCustomizationArgsDictValues dict to SubtitledUnicodeDict. + cust_arg_subtitled_unicode_dict = cast( + state_domain.SubtitledUnicodeDict, cust_arg_value + ) + add_subtitled_unicode_from_dict( + cust_arg_subtitled_unicode_dict + ) + except Exception: + pass + + if isinstance(cust_arg_value, list): + for item in cust_arg_value: + # Each of these conversions are intended to be run on + # every single item. + try: + # Here we use cast because we are narrowing down the + # type UnionOfCustomizationArgsDictValues dict to + # SubtitledHtmlDict. + item_subtitled_html_dict = cast( + state_domain.SubtitledHtmlDict, item + ) + add_subtitled_html_from_dict( + item_subtitled_html_dict + ) + except Exception: + pass + + try: + # Here we use cast because we are narrowing down the + # type UnionOfCustomizationArgsDictValues dict to + # SubtitledUnicodeDict. + item_subtitled_unicode_dict = cast( + state_domain.SubtitledUnicodeDict, item + ) + add_subtitled_unicode_from_dict( + item_subtitled_unicode_dict + ) + except Exception: + pass + + return content_id_to_content_value + + # Repository SAVE and DELETE methods. def apply_change_list( exploration_id: str, change_list: Sequence[exp_domain.ExplorationChange] diff --git a/core/domain/exp_services_test.py b/core/domain/exp_services_test.py index f4ab1f23c216f..81dd09e4da89b 100644 --- a/core/domain/exp_services_test.py +++ b/core/domain/exp_services_test.py @@ -2404,6 +2404,282 @@ def test_cannot_load_yaml_with_no_schema_version(self) -> None: ) +class GetContentUpdatesFromCmdEditStatePropertyChangeTests( + test_utils.GenericTestBase +): + """Tests for get_content_updates_from_cmd_edit_state_property_change.""" + + def test_returns_empty_mapping_for_non_edit_state_property_cmd( + self, + ) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_ADD_STATE, + 'state_name': 'State A', + 'content_id_for_state_content': 'content_1', + 'content_id_for_default_outcome': 'default_outcome_1', + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {}, + ) + + def test_returns_empty_mapping_when_new_value_is_none(self) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'new_value': None, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {}, + ) + + def test_extracts_content_updates_for_content_property(self) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'new_value': { + 'content_id': 'content_1', + 'html': 'New content.
', + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {'content_1': 'New content.
'}, + ) + + def test_extracts_content_updates_for_default_outcome(self) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': ( + exp_domain.STATE_PROPERTY_INTERACTION_DEFAULT_OUTCOME + ), + 'new_value': { + 'feedback': { + 'content_id': 'default_outcome_1', + 'html': 'Try again.
', + } + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {'default_outcome_1': 'Try again.
'}, + ) + + def test_extracts_content_updates_for_answer_groups(self) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': ( + exp_domain.STATE_PROPERTY_INTERACTION_ANSWER_GROUPS + ), + 'new_value': [ + { + 'outcome': { + 'feedback': { + 'content_id': 'answer_group_1', + 'html': 'Correct.
', + } + } + }, + { + 'outcome': { + 'feedback': { + 'content_id': 'answer_group_2', + 'html': 'Incorrect.
', + } + } + }, + ], + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + { + 'answer_group_1': 'Correct.
', + 'answer_group_2': 'Incorrect.
', + }, + ) + + def test_extracts_content_updates_for_hints_and_solution(self) -> None: + hints_change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': exp_domain.STATE_PROPERTY_INTERACTION_HINTS, + 'new_value': [ + { + 'hint_content': { + 'content_id': 'hint_1', + 'html': 'Hint 1.
', + } + }, + { + 'hint_content': { + 'content_id': 'hint_2', + 'html': 'Hint 2.
', + } + }, + ], + } + ) + solution_change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': exp_domain.STATE_PROPERTY_INTERACTION_SOLUTION, + 'new_value': { + 'explanation': { + 'content_id': 'solution_1', + 'html': 'Explanation.
', + } + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + hints_change + ), + { + 'hint_1': 'Hint 1.
', + 'hint_2': 'Hint 2.
', + }, + ) + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + solution_change + ), + {'solution_1': 'Explanation.
'}, + ) + + def test_extracts_content_updates_for_customization_args(self) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': ( + exp_domain.STATE_PROPERTY_INTERACTION_CUST_ARGS + ), + 'new_value': { + 'placeholder': { + 'value': { + 'content_id': 'ca_placeholder_1', + 'unicode_str': 'Enter answer', + } + }, + 'choices': { + 'value': [ + { + 'content_id': 'ca_choices_1', + 'html': 'Choice 1
', + }, + { + 'content_id': 'ca_choices_2', + 'unicode_str': 'Choice 2
', + }, + ] + }, + 'rows': {'value': 1}, + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + { + 'ca_placeholder_1': 'Enter answer', + 'ca_choices_1': 'Choice 1
', + 'ca_choices_2': 'Choice 2
', + }, + ) + + def test_extracts_content_updates_for_customization_args_html_dict( + self, + ) -> None: + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': ( + exp_domain.STATE_PROPERTY_INTERACTION_CUST_ARGS + ), + 'new_value': { + 'question': { + 'value': { + 'content_id': 'ca_question_1', + 'html': 'Question prompt
', + } + }, + 'rows': {'value': 2}, + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {'ca_question_1': 'Question prompt
'}, + ) + + def test_should_not_extract_content_for_invalid_customization_args( + self, + ) -> None: + # Invalid value type for customization arg. + change = exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'state_name': 'State A', + 'property_name': ( + exp_domain.STATE_PROPERTY_INTERACTION_CUST_ARGS + ), + 'new_value': { + 'question': { + 'value': [999], + }, + 'rows': {'value': 2}, + }, + } + ) + + self.assertEqual( + exp_services.get_content_updates_from_cmd_edit_state_property_change( + change + ), + {}, + ) + + class GetImageFilenamesFromExplorationTests(ExplorationServicesUnitTests): def test_get_image_filenames_from_exploration(self) -> None: diff --git a/core/domain/fs_services.py b/core/domain/fs_services.py index db651303fc034..cac189255fdb4 100644 --- a/core/domain/fs_services.py +++ b/core/domain/fs_services.py @@ -114,8 +114,25 @@ class GcsFileSystem(GeneralFileSystem): This implementation ignores versioning. """ - def __init__(self, entity_name: str, entity_id: str) -> None: - self._bucket_name = app_identity_services.get_gcs_resource_bucket_name() + def __init__( + self, + entity_name: str, + entity_id: str, + oppia_project_id: Optional[str] = None, + ) -> None: + """Constructs a GcsFileSystem object. + + Args: + entity_name: str. The name of the entity + (eg: exploration, topic etc). + entity_id: str. The ID of the corresponding entity. + oppia_project_id: Optional[str]. The Google Cloud Project ID. Explicitly + required when running on Beam Dataflow, as workers cannot + retrieve the ID from environment variables. + """ + self._bucket_name = app_identity_services.get_gcs_resource_bucket_name( + oppia_project_id + ) super().__init__(entity_name, entity_id) def _get_gcs_file_url(self, filepath: str) -> str: diff --git a/core/domain/html_cleaner_test.py b/core/domain/html_cleaner_test.py index 420cb99a66a02..e09e6bd4f6b7c 100644 --- a/core/domain/html_cleaner_test.py +++ b/core/domain/html_cleaner_test.py @@ -18,6 +18,7 @@ from __future__ import annotations +from core import utils from core.domain import html_cleaner from core.tests import test_utils @@ -50,6 +51,36 @@ def test_whitelisted_tags(self) -> None: ): html_cleaner.filter_a('link', 'href', 'http://www.oppia.com') + def test_filter_a_with_https_scheme(self) -> None: + """Test filter_a allows https URLs.""" + self.assertTrue( + html_cleaner.filter_a('a', 'href', 'https://www.oppia.com') + ) + + def test_filter_a_with_target_attribute(self) -> None: + """Test filter_a allows target attribute.""" + self.assertTrue(html_cleaner.filter_a('a', 'target', '_blank')) + + def test_filter_a_with_invalid_scheme(self) -> None: + """Test filter_a rejects non-http/https schemes like ftp.""" + self.assertFalse( + html_cleaner.filter_a('a', 'href', 'ftp://www.oppia.com') + ) + + def test_filter_a_with_javascript_scheme(self) -> None: + """Test filter_a rejects javascript scheme.""" + self.assertFalse( + html_cleaner.filter_a('a', 'href', 'javascript:alert(1)') + ) + + def test_filter_a_with_unknown_attribute(self) -> None: + """Test filter_a returns False for unknown attributes.""" + self.assertFalse(html_cleaner.filter_a('a', 'onclick', 'alert(1)')) + + def test_filter_a_with_empty_href(self) -> None: + """Test filter_a rejects empty href.""" + self.assertFalse(html_cleaner.filter_a('a', 'href', '')) + def test_good_tags_allowed(self) -> None: test_data: List[Tuple[str, str]] = [ ( @@ -165,6 +196,19 @@ def test_strip_html_tags(self) -> None: for datum in test_data: self.assertEqual(html_cleaner.strip_html_tags(datum[0]), datum[1]) + def test_strip_html_tags_removes_all_markup(self) -> None: + """Test that strip_html_tags removes nested and complex tags.""" + self.assertEqual( + html_cleaner.strip_html_tags( + 'Hello World
Just text
'] + ), + [], + ) + + def test_get_image_filenames_from_html_strings_with_empty_list( + self, + ) -> None: + """Test get_image_filenames_from_html_strings with empty list.""" + self.assertEqual( + html_cleaner.get_image_filenames_from_html_strings([]), + [], + ) + + def test_get_image_filenames_from_html_strings_with_only_images( + self, + ) -> None: + """Test get_image_filenames_from_html_strings with only image tags.""" + html_strings = [ + '')) + self.assertTrue(html_cleaner.is_html_empty('')) + self.assertTrue(html_cleaner.is_html_empty('')) + self.assertTrue(html_cleaner.is_html_empty('')) + self.assertTrue(html_cleaner.is_html_empty('')) + self.assertTrue(html_cleaner.is_html_empty('
Hello
')) + self.assertFalse(html_cleaner.is_html_empty('Some text')) + + def test_empty_string_is_empty(self) -> None: + """Test that an empty string is considered empty.""" + self.assertTrue(html_cleaner.is_html_empty('')) + + def test_whitespace_only_string_is_empty(self) -> None: + """Test that whitespace only string is considered empty.""" + self.assertTrue(html_cleaner.is_html_empty(' ')) + + def test_double_quotes_only_is_empty(self) -> None: + """Test that \"\" is considered empty.""" + self.assertTrue(html_cleaner.is_html_empty('\"\"')) + + def test_single_quotes_only_is_empty(self) -> None: + """Test that '' is considered empty.""" + self.assertTrue(html_cleaner.is_html_empty('\'\'')) + + +class ValidateRteTagsTests(test_utils.GenericTestBase): + """Tests for the validate_rte_tags function.""" + + def test_valid_html_without_rte_tags_passes(self) -> None: + """Test that plain HTML without RTE tags passes validation.""" + html_cleaner.validate_rte_tags('Hello world
') + + def test_image_missing_alt_attribute_raises_error(self) -> None: + """Test image tag without alt-with-value raises error.""" + html_data = ( + 'Plain text
', + is_tag_nested_inside_tabs_or_collapsible=True, + ) + + +class ValidateTabsAndCollapsibleRteTagsTests(test_utils.GenericTestBase): + """Tests for validate_tabs_and_collapsible_rte_tags function.""" + + def test_no_tabs_or_collapsible_passes(self) -> None: + """Test that HTML without tabs or collapsible passes.""" + html_cleaner.validate_tabs_and_collapsible_rte_tags( + 'Hello world
' + ) + + def test_tabs_missing_tab_contents_attribute_raises_error(self) -> None: + """Test tabs tag without tab_contents-with-value raises error.""" + html_data = 'This is a test text
' + exploration_id = 'exp_id' + language_accent_code = 'en-US' + filename = 'content_0-en-US-asdjytdyop.mp3' + + def _mock_regenerate_speech_from_text( + _text: str, + _language_accent_code: str, + _oppia_project_id: Optional[str] = None, + ) -> Tuple[bytes, List[Dict[str, Union[str, float]]], Optional[str]]: + + return (b'', [], '') + + with self.swap( + dev_mode_speech_synthesis_services, + 'regenerate_speech_from_text', + _mock_regenerate_speech_from_text, + ): + audio_offset_list = voiceover_regeneration_services.synthesize_voiceover_for_html_string( + exploration_id, content_html, language_accent_code, filename + ) + self.assertEqual(audio_offset_list, []) + @mock.patch( 'core.domain.fs_services.GcsFileSystem.get', side_effect=Exception('Mocked exception during voiceover retrieval'), @@ -419,6 +444,58 @@ def test_should_raise_exception_if_regeneration_failed( ) ) + def test_should_get_empty_audio_sucessfully_in_sync(self) -> None: + content_html = '' + exploration_id = 'exp_id' + language_accent_code = 'en-US' + filename = 'content_0-en-US-asdjytdyop.mp3' + + def _mock_regenerate_speech_from_text( + _text: str, + _language_accent_code: str, + _oppia_project_id: Optional[str] = None, + ) -> Tuple[bytes, List[Dict[str, Union[str, float]]], Optional[str]]: + + return (b'', [], '') + + with self.swap( + dev_mode_speech_synthesis_services, + 'regenerate_speech_from_text', + _mock_regenerate_speech_from_text, + ): + audio_offset_list = voiceover_regeneration_services.synthesize_voiceover_for_html_string( + exploration_id, content_html, language_accent_code, filename + ) + self.assertEqual(audio_offset_list, []) + + def test_should_get_empty_audio_sucessfully_in_async(self) -> None: + content_html = '' + exploration_id = 'exp_id' + language_accent_code = 'en-US' + filename = 'content_0-en-US-asdjytdyop.mp3' + + def _mock_regenerate_speech_from_text( + _text: str, + _language_accent_code: str, + _oppia_project_id: Optional[str] = None, + ) -> Tuple[bytes, List[Dict[str, Union[str, float]]], Optional[str]]: + + return (b'', [], '') + + with self.swap( + dev_mode_speech_synthesis_services, + 'regenerate_speech_from_text', + _mock_regenerate_speech_from_text, + ): + audio_offset_list = voiceover_regeneration_services.synthesize_voiceover_for_html_string( + exploration_id, + content_html, + language_accent_code, + filename, + 'dev-project-id', + ) + self.assertEqual(audio_offset_list, []) + def test_should_be_able_to_get_new_voiceover_filename(self) -> None: content_id = 'content_0' language_accent_code = 'en-US' diff --git a/core/domain/voiceover_services.py b/core/domain/voiceover_services.py index 23376eaf43d08..5bb44860917bd 100644 --- a/core/domain/voiceover_services.py +++ b/core/domain/voiceover_services.py @@ -18,17 +18,20 @@ from __future__ import annotations +import collections import datetime import json -import os +import logging -from core import feconf -from core.constants import constants +from core import constants, feconf from core.domain import ( + cloud_task_domain, email_manager, exp_domain, exp_fetchers, + exp_services, state_domain, + suggestion_services, taskqueue_services, translation_domain, translation_fetchers, @@ -40,7 +43,7 @@ from core.platform import models from core.storage.voiceover import gae_models -from typing import Dict, List, Optional, Tuple, cast +from typing import Dict, List, Optional, Set, Tuple, cast MYPY = False if MYPY: # pragma: no cover @@ -57,7 +60,7 @@ MAX_SAMPLE_VOICEOVERS_FOR_GIVEN_VOICE_ARTIST = 5 -def _get_entity_voiceovers_from_model( +def get_entity_voiceovers_from_model( entity_voiceovers_model: voiceover_models.EntityVoiceoversModel, ) -> voiceover_domain.EntityVoiceovers: """Returns the EntityVoiceovers domain object from its model representation @@ -108,7 +111,7 @@ def get_voiceovers_for_given_language_accent_code( ) if entity_voiceovers_model: - return _get_entity_voiceovers_from_model(entity_voiceovers_model) + return get_entity_voiceovers_from_model(entity_voiceovers_model) return voiceover_domain.EntityVoiceovers.create_empty( entity_type=entity_type, entity_id=entity_id, @@ -141,7 +144,7 @@ def get_entity_voiceovers_for_given_exploration( for model_instance in entity_voiceovers_models: entity_voiceovers_objects.append( - _get_entity_voiceovers_from_model(model_instance) + get_entity_voiceovers_from_model(model_instance) ) return entity_voiceovers_objects @@ -305,7 +308,7 @@ def compute_voiceover_related_change( # English content was modified, so all associated # voiceovers must be marked as needing update. if ( - language_code != constants.DEFAULT_LANGUAGE_CODE + language_code != constants.constants.DEFAULT_LANGUAGE_CODE and entity_voiceovers.language_accent_code not in language_accent_codes ): @@ -332,7 +335,7 @@ def compute_voiceover_related_change( # English content was modified, so all associated # voiceovers must be removed. if ( - language_code != constants.DEFAULT_LANGUAGE_CODE + language_code != constants.constants.DEFAULT_LANGUAGE_CODE and entity_voiceovers.language_accent_code not in language_accent_codes ): @@ -554,6 +557,97 @@ def save_language_accent_support( voiceover_autogeneration_policy_model.put() +def is_accent_code_valid_for_autogeneration(language_accent_code: str) -> bool: + """The method validates whether the provided language accent code is valid + for Oppia's voiceover autogeneration. + + Args: + language_accent_code: str. The language accent code to be validated. + + Returns: + bool. True if the provided language accent code is valid for Oppia's + voiceover autogeneration, False otherwise. + """ + autogeneratable_language_accents = ( + get_autogeneratable_language_accent_codes() + ) + return ( + isinstance(language_accent_code, str) + and language_accent_code in autogeneratable_language_accents + ) + + +def get_new_auto_voiceover_accent( + updated_language_accent_mapping: Dict[str, Dict[str, bool]], +) -> Optional[str]: + """Returns the newly added language-accent code enabled for automatic + voiceover regeneration, if any. + + Args: + updated_language_accent_mapping: dict(str, dict(str, bool)). Mapping of + language codes to their accent configurations after the update. + Each accent code maps to a boolean indicating whether automatic + voiceover generation is enabled. + + Returns: + Optional[str]. The newly added language-accent code enabled for automatic + voiceover regeneration, or None if no such accent was added. + """ + retrieved_voiceover_autogeneration_policy_model = ( + voiceover_models.VoiceoverAutogenerationPolicyModel.get( + voiceover_models.VOICEOVER_AUTOGENERATION_POLICY_ID, strict=False + ) + ) + voiceover_autogeneration_policy_model = ( + retrieved_voiceover_autogeneration_policy_model + if retrieved_voiceover_autogeneration_policy_model is not None + else voiceover_models.VoiceoverAutogenerationPolicyModel( + id=voiceover_models.VOICEOVER_AUTOGENERATION_POLICY_ID + ) + ) + + existing_language_accent_mapping = ( + voiceover_autogeneration_policy_model.language_codes_mapping + if voiceover_autogeneration_policy_model.language_codes_mapping + is not None + else {} + ) + existing_autogeneratable_accents: Set[str] = set() + updated_autogeneratable_accents: Set[str] = set() + + for accent_mapping in existing_language_accent_mapping.values(): + for ( + language_accent_code, + supports_autogeneration, + ) in accent_mapping.items(): + if supports_autogeneration: + existing_autogeneratable_accents.add(language_accent_code) + + for accent_mapping in updated_language_accent_mapping.values(): + for ( + language_accent_code, + supports_autogeneration, + ) in accent_mapping.items(): + if supports_autogeneration: + updated_autogeneratable_accents.add(language_accent_code) + + new_accents_set = ( + updated_autogeneratable_accents - existing_autogeneratable_accents + ) + + # Since the UI triggers a backend request immediately whenever a language + # accent code is updated, the new_accents_set can contain at most one element. + # Therefore, we can safely use pop() to retrieve the newly added language + # accent code. + if new_accents_set: + assert len(new_accents_set) == 1, ( + 'Expected only one new language-accent code to be added for automatic ' + 'voiceover regeneration, but found multiple: %s' % new_accents_set + ) + return new_accents_set.pop() + return None + + def get_language_accent_master_list() -> Dict[str, Dict[str, str]]: """The method returns the lanaguage accent master list stored in the JSON file. @@ -565,14 +659,10 @@ def get_language_accent_master_list() -> Dict[str, Dict[str, str]]: language-accent pairs that Oppia may support for voiceovers (manual and auto). """ - file_path = os.path.join( - feconf.VOICEOVERS_DATA_DIR, 'language_accent_master_list.json' + language_accent_master_list: Dict[str, Dict[str, str]] = ( + constants.language_accent_master_list_constants ) - with open(file_path, 'r', encoding='utf-8') as f: - language_accent_master_list: Dict[str, Dict[str, str]] = json.loads( - f.read() - ) - return language_accent_master_list + return language_accent_master_list def get_language_accent_codes_to_descriptions() -> Dict[str, str]: @@ -633,14 +723,10 @@ def get_autogeneratable_language_accent_list() -> Dict[str, Dict[str, str]]: for voiceover generation, while 'voice_code' signifies the desired voice type. """ - file_path = os.path.join( - feconf.VOICEOVERS_DATA_DIR, 'autogeneratable_language_accent_list.json' + autogeneratable_language_accent_list: Dict[str, Dict[str, str]] = ( + constants.autogeneratable_language_accent_constants ) - with open(file_path, 'r', encoding='utf-8') as f: - autogeneratable_language_accent_list: Dict[str, Dict[str, str]] = ( - json.loads(f.read()) - ) - return autogeneratable_language_accent_list + return autogeneratable_language_accent_list def get_autogeneratable_language_accent_codes() -> List[str]: @@ -805,29 +891,6 @@ def send_email_to_voiceover_admins_and_tech_leads_after_regeneration( ) -def _remove_empty_contents_for_voiceover_regeneration( - language_code_to_contents_mapping: Dict[str, Dict[str, str]], -) -> None: - """Removes empty contents from the provided input. - - Args: - language_code_to_contents_mapping: dict. A dictionary mapping language - codes to the corresponding content IDs and their associated HTML - that require voiceover regeneration. - """ - for ( - _, - content_ids_to_content_values, - ) in language_code_to_contents_mapping.items(): - content_ids_to_remove = [ - content_id - for content_id, html in (content_ids_to_content_values.items()) - if not html.strip() - ] - for content_id in content_ids_to_remove: - del content_ids_to_content_values[content_id] - - def extract_english_voiceover_texts_from_exploration( exploration: exp_domain.Exploration, ) -> Dict[str, Dict[str, str]]: @@ -906,68 +969,35 @@ def extract_translated_voiceover_texts_from_entity_translations( return language_code_to_contents_mapping -def _regenerate_voiceovers_for_given_contents( +def regenerate_voiceovers_for_given_contents( exploration_id: str, - exploration_title: str, exploration_version: int, language_code_to_contents_mapping: Dict[str, Dict[str, str]], - date_time: str, - author_id: str, + task_run_id: str, specific_language_accent_code: Optional[str] = None, - task_run_id: Optional[str] = None, ) -> None: - """Private helper method to regenerate voiceovers for specified contents + """Helper method to regenerate voiceovers for specified contents of an exploration. Args: exploration_id: str. The ID of the exploration for which voiceovers need to be regenerated. - exploration_title: str. The title of the exploration for which - voiceovers need to be regenerated. exploration_version: int. The version of the exploration for which voiceovers need to be regenerated. language_code_to_contents_mapping: dict. A dictionary mapping language codes to the corresponding content IDs and their associated HTML that require voiceover regeneration. - date_time: str. The ISO-formatted timestamp indicating when the - regeneration process was initiated. - author_id: str. The ID of the user who triggered the voiceover - regeneration, either directly or indirectly. + task_run_id: str. The unique identifier for the voiceover + regeneration task. specific_language_accent_code: Optional[str]. The specific language accent code to use for voiceover regeneration, if provided. - task_run_id: str|None. The unique identifier for the voiceover - regeneration task. If None, the method is invoked by a - synchronous process and task-tracking is not required. """ - # A dictionary mapping each language code to a list of accent codes that - # support autogeneration. - language_code_to_autogeneratable_accent_codes = {} - - # Remove empty contents from the voiceover regeneration mapping. - _remove_empty_contents_for_voiceover_regeneration( - language_code_to_contents_mapping - ) - - # A list of error collections that occurred during the - # voiceover regeneration. - error_collections_during_voiceover_regeneration: List[ - Dict[str, List[Tuple[str, str]] | str] - ] = [] - # Get all language codes that need voiceover regeneration in this request. language_codes = list(language_code_to_contents_mapping.keys()) - language_accent_codes_to_descriptions = ( - get_language_accent_codes_to_descriptions() - ) - - # Counter to track the number of contents for which voiceover regeneration - # is triggered. - number_of_contents_for_voiceover_regeneration = 0 - - # Counter to track the number of contents that failed to regenerate - # voiceovers. - number_of_contents_failed_to_regenerate = 0 + # A dictionary mapping each language code to a list of accent codes that + # support autogeneration. + language_code_to_autogeneratable_accent_codes = {} # Retrieve all Oppia-supported language accents, grouped by language code, # for which voiceovers need to be regenerated for the given contents. @@ -975,158 +1005,436 @@ def _regenerate_voiceovers_for_given_contents( language_accent_codes = ( get_supported_autogeneratable_accents_by_language(language_code) ) + if not language_accent_codes: continue + + if specific_language_accent_code: + language_code_to_autogeneratable_accent_codes[language_code] = [ + specific_language_accent_code + ] + break + language_code_to_autogeneratable_accent_codes[language_code] = ( language_accent_codes ) - # A list of language accents for which voiceovers are regenerated. - language_accents_used_for_voiceover_regeneration = [] + voiceover_regeneration_job = voiceover_cloud_task_services.create_voiceover_regeneration_task_with_status_generating( + exploration_id, + task_run_id, + language_code_to_contents_mapping, + language_code_to_autogeneratable_accent_codes, + ) - requested_task_is_async: bool = task_run_id is not None + # Ruling out the possibility of None for mypy type checking. + assert voiceover_regeneration_job is not None - if requested_task_is_async: - # Ruling out the possibility of None for mypy type checking. - assert task_run_id is not None - voiceover_regeneration_task = ( - voiceover_cloud_task_services.get_voiceover_regeneration_task( - exploration_id, task_run_id - ) - ) + voiceover_cloud_task_services.save_voiceover_regeneration_job( + voiceover_regeneration_job + ) - if requested_task_is_async and voiceover_regeneration_task is None: - # Ruling out the possibility of None for mypy type checking. - assert task_run_id is not None - voiceover_regeneration_task = voiceover_cloud_task_services.create_voiceover_regeneration_task_with_status_generating( - exploration_id, - task_run_id, - language_code_to_contents_mapping, - language_code_to_autogeneratable_accent_codes, - ) + # Voiceover regeneration for a large number of contents within a single + # Cloud Task run (deferred request) significantly increases the workload and + # may lead to timeout failures due to Gunicorn limitations of 60 seconds. + # To mitigate this issue, a single deferred regeneration task is split into + # multiple smaller batches. + divide_and_enqueue_voiceover_regeneration_tasks_in_smaller_batches( + language_code_to_contents_mapping, + language_code_to_autogeneratable_accent_codes, + exploration_id, + exploration_version, + task_run_id, + ) - # Ruling out the possibility of None for mypy type checking. - assert voiceover_regeneration_task is not None - voiceover_cloud_task_services.save_voiceover_regeneration_task_run_mapping( - voiceover_regeneration_task +def divide_and_enqueue_voiceover_regeneration_tasks_in_smaller_batches( + language_code_to_contents_mapping: Dict[str, Dict[str, str]], + language_code_to_autogeneratable_accent_codes: Dict[str, List[str]], + exploration_id: str, + exploration_version: int, + parent_cloud_task_run_id: str, +) -> None: + """It divides the voiceover regeneration process for an exploration into + smaller batches and enqueues a separate task for each batch in the + Google Cloud Task Queue. This approach prevents asynchronous deferred + requests from timing out when processing a large number of contents + in one request, thereby avoiding the 60-second Gunicorn timeout limit. + + Args: + language_code_to_contents_mapping: dict. A dictionary mapping language + codes to the corresponding content IDs and their associated HTML + that require voiceover regeneration. + language_code_to_autogeneratable_accent_codes: dict. A dictionary mapping + language codes to a list of accent codes that support autogeneration. + exploration_id: str. The ID of the exploration for which voiceovers + need to be regenerated. + exploration_version: int. The version of the exploration for which + voiceovers need to be regenerated. + parent_cloud_task_run_id: str. The unique identifier for the parent + cloud task run, which is responsible for regenerating voiceovers + for all the contents of the exploration in batches. + """ + logging.info( + 'Voiceover regeneration logs: Starting to divide and enqueue voiceover ' + 'regeneration tasks in smaller batches for exploration_id: %s, ' + 'parent_cloud_task_run_id: %s' + % ( + exploration_id, + parent_cloud_task_run_id, ) + ) + # Based on testing data, regenerating a voiceover for each state content + # takes approximately 6 seconds. Therefore, to avoid hitting the timeout + # limit, we can process about 8 contents per batch. This would take roughly + # 48 seconds, leaving sufficient buffer time to handle any variations + # in processing. + batch_size = 8 - errors_while_voiceover_regeneration = [] + batch_counter = 0 + child_cloud_task_model_ids = [] - for language_code in language_codes: + for ( + language_code, + content_ids_to_content_values, + ) in language_code_to_contents_mapping.items(): language_accent_codes = ( language_code_to_autogeneratable_accent_codes.get(language_code, []) ) + for language_accent_code in language_accent_codes: + content_id_value_pairs = list(content_ids_to_content_values.items()) + + for i in range(0, len(content_id_value_pairs), batch_size): + batch_content_id_value_pairs = content_id_value_pairs[ + i : i + batch_size + ] + batch_content_ids_to_content_values = dict( + batch_content_id_value_pairs + ) + batch_counter += 1 + + child_cloud_task_model_id = ( + taskqueue_services.get_new_cloud_task_run_id() + ) + + logging.info( + 'Voiceover regeneration logs: Enqueuing batch %d for ' + 'exploration_id: %s, parent_cloud_task_run_id: %s, ' + 'child_cloud_task_run_id: %s' + % ( + batch_counter, + exploration_id, + parent_cloud_task_run_id, + child_cloud_task_model_id, + ) + ) - content_ids_to_content_values = language_code_to_contents_mapping.get( - language_code, {} + voiceover_regeneration_task_batch_instance = ( + cloud_task_domain.VoiceoverRegenerationTaskBatch( + parent_cloud_task_run_id, + child_cloud_task_model_id, + exploration_id, + exploration_version, + language_accent_code, + batch_content_ids_to_content_values, + ) + ) + + voiceover_cloud_task_services.create_voiceover_regeneration_task_batch_model( + voiceover_regeneration_task_batch_instance + ) + + # Enqueue to Google cloud task queue. + taskqueue_services.defer_voiceover_regeneration_task_in_batches( + feconf.FUNCTION_ID_TO_FUNCTION_NAME_FOR_DEFERRED_JOBS[ + 'FUNCTION_ID_REGENERATE_VOICEOVERS_FOR_BATCH_CONTENTS' + ], + taskqueue_services.QUEUE_NAME_VOICEOVER_REGENERATION, + parent_cloud_task_run_id, + child_cloud_task_model_id, + exploration_id, + ) + + child_cloud_task_model_ids.append(child_cloud_task_model_id) + + logging.info( + 'Voiceover regeneration logs: Number of batches: %s, Parent Cloud Task Run ID: %s, Child Cloud Task Run IDs: %s' + % ( + batch_counter, + parent_cloud_task_run_id, + child_cloud_task_model_ids, ) + ) + + +def regenerate_voiceovers_for_batch_contents( + exploration_id: str, + parent_cloud_task_run_id: str, + child_cloud_task_run_id: str, +) -> None: + """Regenerates automatic voiceovers for some contents of an exploration, so + that we can't hit the state where in an async deferred request due to large + numbers of contents to regenerate in one go we have a timeout becuase of + Gunicorn's timeout of 60 secs. + + Args: + exploration_id: str. The ID of the exploration for which voiceovers + need to be regenerated. + parent_cloud_task_run_id: str. The unique identifier for the parent + cloud task run, which is responsible for regenerating voiceovers + for all the contents of the exploration in batches. + child_cloud_task_run_id: str. The unique identifier for the child + cloud task run, which is responsible for regenerating voiceovers + for a batch of contents of the exploration in a language accent. + + Raises: + Exception. Raised when there is an error during the voiceover + regeneration process for the batch of contents. + """ + logging.info( + 'Voiceover regeneration logs: Starting to regenerate voiceovers for ' + 'batch contents for exploration_id: %s, parent_cloud_task_run_id: %s, ' + 'child_cloud_task_run_id: %s' + % (exploration_id, parent_cloud_task_run_id, child_cloud_task_run_id) + ) + + voiceover_regeneration_batch_execution_job = voiceover_cloud_task_services.get_voiceover_regeneration_task_batch_model( + parent_cloud_task_run_id, child_cloud_task_run_id + ) + + logging.info( + 'Voiceover regeneration logs: Trying to fetch voiceover regeneration batch execution job, ' + 'parent_cloud_task_run_id: %s, child_cloud_task_run_id: %s.' + % (parent_cloud_task_run_id, child_cloud_task_run_id) + ) + + # Ruling out the possibility of None for mypy type checking. + assert voiceover_regeneration_batch_execution_job is not None + + exploration_id = voiceover_regeneration_batch_execution_job.exploration_id + exploration_version = ( + voiceover_regeneration_batch_execution_job.exploration_version + ) + language_accent_code = ( + voiceover_regeneration_batch_execution_job.language_accent_code + ) + content_ids_to_content_values = ( + voiceover_regeneration_batch_execution_job.content_ids_to_contents_map + ) + + try: + errors_while_voiceover_regeneration = voiceover_regeneration_services.regenerate_voiceovers_of_exploration( + exploration_id, + exploration_version, + content_ids_to_content_values, + language_accent_code, + ) + except Exception as e: + errors_while_voiceover_regeneration = [ + (content_id, str(e)) + for content_id in content_ids_to_content_values.keys() + ] + + error_collections_during_voiceover_regeneration = [] + + error_collections_during_voiceover_regeneration.append( + json.dumps( + { + 'exploration_id': exploration_id, + 'language_accent_code': language_accent_code, + 'error_messages': errors_while_voiceover_regeneration, + } + ) + ) + + child_cloud_task_run = taskqueue_services.get_cloud_task_run_by_model_id( + child_cloud_task_run_id + ) + # Ruling out the possibility of None for mypy type checking. + assert child_cloud_task_run is not None + + child_cloud_task_run.exception_messages_for_failed_runs.extend( + error_collections_during_voiceover_regeneration + ) + if len(errors_while_voiceover_regeneration) > 0: + child_cloud_task_run.latest_job_state = 'PERMANENTLY_FAILED' + else: + child_cloud_task_run.latest_job_state = 'SUCCEEDED' + + taskqueue_services.update_cloud_task_run_model(child_cloud_task_run) + + wrap_up_voiceover_regeneration_task( + exploration_id, parent_cloud_task_run_id + ) - for language_accent_code in language_accent_codes: - if ( - specific_language_accent_code is not None - and language_accent_code != specific_language_accent_code - ): - continue - language_accents_used_for_voiceover_regeneration.append( - language_accent_codes_to_descriptions.get( - language_accent_code, '' +def wrap_up_voiceover_regeneration_task( + exploration_id: str, + parent_cloud_task_run_id: str, +) -> None: + """Wraps up the voiceover regeneration task by sending a summary email to + voiceover admins and tech leads, which includes the details of the + voiceover regeneration process. + + Args: + exploration_id: str. The ID of the exploration for which voiceovers + were regenerated. + parent_cloud_task_run_id: str. The unique identifier for the parent + cloud task run, which is responsible for regenerating voiceovers + for all the contents of the exploration in batches. + """ + child_cloud_task_run_ids = [] + language_accent_codes = [] + number_of_contents_for_voiceover_regeneration = 0 + + voiceover_regeneration_batch_instances = voiceover_cloud_task_services.get_voiceover_regeneration_batch_instances_by_parent_task_run_id( + parent_cloud_task_run_id + ) + + for batch_instance in voiceover_regeneration_batch_instances: + child_cloud_task_run_ids.append(batch_instance.child_cloud_task_run_id) + language_accent_codes.append(batch_instance.language_accent_code) + number_of_contents_for_voiceover_regeneration += len( + batch_instance.content_ids_to_contents_map + ) + + child_cloud_task_runs = taskqueue_services.get_cloud_task_runs_by_model_ids( + child_cloud_task_run_ids + ) + + # Verify first if all the task runs are completed i.e., their status must + # be either 'SUCCEEDED' or 'PERMANENTLY_FAILED'. If not, we should not + # proceed with wrapping up the voiceover regeneration task, as it indicates + # that some batches are still being processed. + for child_cloud_task_run in child_cloud_task_runs: + if child_cloud_task_run.latest_job_state not in [ + 'SUCCEEDED', + 'PERMANENTLY_FAILED', + ]: + logging.info( + 'Voiceover regeneration logs: Not wrapping up the voiceover ' + 'regeneration task for parent_cloud_task_run_id: %s, because ' + 'child_cloud_task_run_id: %s is still in processing with status: %s' + % ( + parent_cloud_task_run_id, + child_cloud_task_run.task_id, + child_cloud_task_run.latest_job_state, ) ) + return - number_of_contents_for_voiceover_regeneration += len( - content_ids_to_content_values - ) + error_collections_during_voiceover_regeneration: List[ + Dict[str, str | List[Tuple[str, str]]] + ] = [] + language_accent_code_to_error: Dict[str, List[Tuple[str, str]]] = ( + collections.defaultdict(list) + ) + number_of_contents_failed_to_regenerate = 0 - errors_while_voiceover_regeneration = voiceover_regeneration_services.regenerate_voiceovers_of_exploration( - exploration_id, - exploration_version, - content_ids_to_content_values, - language_accent_code, + voiceover_regeneration_job_status = ( + voiceover_cloud_task_services.get_voiceover_regeneration_job( + exploration_id, parent_cloud_task_run_id + ) + ) + # Ruling out the possibility of None for mypy type checking. + assert voiceover_regeneration_job_status is not None + + parent_cloud_task_run = taskqueue_services.get_cloud_task_run_by_model_id( + parent_cloud_task_run_id + ) + # Ruling out the possibility of None for mypy type checking. + assert parent_cloud_task_run is not None + + for cloud_task_run in child_cloud_task_runs: + for error_details in cloud_task_run.exception_messages_for_failed_runs: + error_collections_during_voiceover_regeneration.append( + json.loads(error_details) ) - failed_content_ids = [ - error[0] for error in errors_while_voiceover_regeneration - ] + final_error_string = 'Exploration ID: %s\n' % exploration_id - if requested_task_is_async: - # Ruling out the possibility of None for mypy type checking. - assert voiceover_regeneration_task is not None - voiceover_regeneration_task.update_final_content_status_for_cloud_task_run( - language_accent_code, failed_content_ids - ) + for error_collection in error_collections_during_voiceover_regeneration: + # Here we use cast because we are narrowing down the type of + # 'language_accent_code' from Union of str and List to a str. + language_accent_code: str = cast( + str, error_collection['language_accent_code'] + ) - if errors_while_voiceover_regeneration: - error_collections_during_voiceover_regeneration.append( - { - 'exploration_id': exploration_id, - 'language_accent_code': language_accent_code, - 'error_messages': errors_while_voiceover_regeneration, - } - ) - number_of_contents_failed_to_regenerate += len( - errors_while_voiceover_regeneration - ) + # Here we use cast because we are narrowing down the type of + # 'error_messages' from Union of str and List to a List of Tuples. + content_id_and_error_message_tuple = cast( + List[Tuple[str, str]], error_collection['error_messages'] + ) - if requested_task_is_async: - # Ruling out the possibility of None for mypy type checking. - assert voiceover_regeneration_task is not None - voiceover_cloud_task_services.save_voiceover_regeneration_task_run_mapping( - voiceover_regeneration_task + language_accent_code_to_error[language_accent_code].extend( + content_id_and_error_message_tuple ) + for ( + language_accent_code, + content_id_and_error_message_tuple, + ) in language_accent_code_to_error.items(): + final_error_string += 'Language Accent Code: %s\nErrors: %s \n' % ( + language_accent_code, + content_id_and_error_message_tuple, + ) + failed_content_ids = [ + error[0] for error in content_id_and_error_message_tuple + ] + voiceover_regeneration_job_status.update_failed_content_status( + language_accent_code, failed_content_ids + ) + number_of_contents_failed_to_regenerate += len(failed_content_ids) + + if number_of_contents_failed_to_regenerate > 0: + parent_cloud_task_run.exception_messages_for_failed_runs.append( + final_error_string + ) + parent_cloud_task_run.latest_job_state = 'PERMANENTLY_FAILED' + taskqueue_services.update_cloud_task_run_model(parent_cloud_task_run) + + voiceover_regeneration_job_status.update_remaining_content_status_as_succeeded() + voiceover_cloud_task_services.save_voiceover_regeneration_job( + voiceover_regeneration_job_status + ) + logging.info( + 'Voiceover regeneration logs: %s' + % voiceover_regeneration_job_status.to_dict() + ) + + exploration = exp_fetchers.get_exploration_by_id( + exploration_id, strict=False + ) + exploration_title = exploration.title if exploration else '' + + language_accent_codes_to_descriptions = ( + get_language_accent_codes_to_descriptions() + ) + language_accent_descriptions_used_for_regeneration = [ + language_accent_codes_to_descriptions.get(language_accent_code, '') + for language_accent_code in language_accent_codes + ] + + logging.info( + 'Voiceover regeneration logs: Finished regenerating voiceovers for ' + 'all batches for exploration_id: %s, now sending summary email to ' + 'voiceover admins and tech leads.' % exploration_id + ) send_email_to_voiceover_admins_and_tech_leads_after_regeneration( exploration_id, exploration_title, - date_time, - language_accents_used_for_voiceover_regeneration, + parent_cloud_task_run.created_on.isoformat(), + language_accent_descriptions_used_for_regeneration, error_collections_during_voiceover_regeneration, number_of_contents_for_voiceover_regeneration, number_of_contents_failed_to_regenerate, - author_id, + feconf.SYSTEM_COMMITTER_ID, ) - if requested_task_is_async: - error_string = '' - for error_collection in error_collections_during_voiceover_regeneration: - error_string += ( - 'Exploration ID: %s\nLanguage Accent Code: %s\nErrors: %s \n' - % ( - error_collection['exploration_id'], - error_collection['language_accent_code'], - error_collection['error_messages'], - ) - ) - - # Ruling out the possibility of None for mypy type checking. - assert task_run_id is not None - cloud_task_run_domain_instance = ( - taskqueue_services.get_cloud_task_run_by_model_id(task_run_id) - ) - # Ruling out the possibility of None for mypy type checking. - assert cloud_task_run_domain_instance is not None - - if errors_while_voiceover_regeneration: - cloud_task_run_domain_instance.latest_job_state = ( - 'PERMANENTLY_FAILED' - ) - cloud_task_run_domain_instance.exception_messages_for_failed_runs.append( - error_string - ) - taskqueue_services.update_cloud_task_run_model( - cloud_task_run_domain_instance - ) - def regenerate_voiceovers_on_exploration_update( exploration_id: str, - exploration_title: str, exploration_version: int, - author_id: str, - date_time: str, - task_run_id: Optional[str] = None, + task_run_id: str, ) -> None: """Regenerates voiceovers for the updated exploration based on the changes made in the exploration content (in English) or translations (in other @@ -1138,20 +1446,21 @@ def regenerate_voiceovers_on_exploration_update( Args: exploration_id: str. The ID of the exploration for which voiceovers need to be regenerated. - exploration_title: str. The title of the exploration. exploration_version: int. The version of the exploration for which voiceovers need to be regenerated. - author_id: str. The ID of the author who made the changes to the - exploration. - date_time: str. The date and time when the changes were - made to the exploration. - task_run_id: str|None. The unique identifier for the voiceover + task_run_id: str. The unique identifier for the voiceover regeneration task. Raises: Exception. If the voiceover regeneration fails for any of the content IDs or language-accent codes. """ + logging.info( + 'Voiceover regeneration logs: Started regenerating voiceovers for ' + 'exploration with ID: %s and version: %s on exploration update.' + % (exploration_id, exploration_version) + ) + # Fetches the exploration change diff for the given exploration ID and # exploration version from the ExplorationCommitLogEntryModel. exploration_commit_log_entry_model_id = 'exploration-%s-%s' % ( @@ -1181,13 +1490,26 @@ def regenerate_voiceovers_on_exploration_update( for change in exploration_change_diff: cmd = change.get('cmd') if cmd == exp_domain.CMD_EDIT_STATE_PROPERTY: - # CMD_EDIT_STATE_PROPERTY is used to fetch the updated content for - # the English language. - updated_content = change['new_value']['html'] - content_id = change['new_value']['content_id'] - language_code_to_contents_mapping.setdefault('en', {})[ - content_id - ] = updated_content + # Here we use cast because the from_dict() method returns a object of + # type BaseChange, which is a parent class for ExplorationChange. + # This cast assures the static type checker that the 'change_object' + # variable is of type ExplorationChange, allowing us to access its + # specific attributes and methods without type errors. + change_object = cast( + exp_domain.ExplorationChange, + exp_domain.ExplorationChange.from_dict(change), + ) + content_id_to_content_values = exp_services.get_content_updates_from_cmd_edit_state_property_change( + change_object + ) + + for ( + content_id, + content_value, + ) in content_id_to_content_values.items(): + language_code_to_contents_mapping.setdefault('en', {})[ + content_id + ] = content_value elif cmd == exp_domain.CMD_EDIT_TRANSLATION: # CMD_EDIT_TRANSLATION is used to fetch the updated content for # the translations in other languages. @@ -1197,23 +1519,20 @@ def regenerate_voiceovers_on_exploration_update( language_code_to_contents_mapping.setdefault(language_code, {})[ content_id ] = updated_content - - _regenerate_voiceovers_for_given_contents( + logging.info( + 'Voiceover regeneration logs: %s' % language_code_to_contents_mapping + ) + regenerate_voiceovers_for_given_contents( exploration_id, - exploration_title, exploration_version, language_code_to_contents_mapping, - date_time, - author_id, - task_run_id=task_run_id, + task_run_id, ) def regenerate_voiceovers_on_exploration_added_to_topic( exploration_id: str, - date_time: str, - author_id: str, - task_run_id: Optional[str] = None, + task_run_id: str, ) -> None: """Regenerates all voiceovers (in English and in all the available translated languages) for the given exploration when it is curated — i.e., @@ -1225,11 +1544,13 @@ def regenerate_voiceovers_on_exploration_added_to_topic( Args: exploration_id: str. The ID of the exploration to regenerate voiceovers for. - date_time: str. The timestamp when the exploration was curated. - author_id: str. The ID of the user who curated the exploration. - task_run_id: str|None. The unique identifier for the voiceover + task_run_id: str. The unique identifier for the voiceover regeneration task. """ + logging.info( + 'Voiceover regeneration logs: Started regenerating voiceovers for ' + 'exploration with ID: %s when it is added to topic.' % exploration_id + ) # A dictionary where each key is a language code, and each value is a # content mapping dictionary. The content mapping dictionary contains # content IDs as keys and their corresponding HTML content as values. @@ -1239,7 +1560,6 @@ def regenerate_voiceovers_on_exploration_added_to_topic( assert exploration is not None exploration_version = exploration.version - exploration_title = exploration.title # Retrieve all English-language contents from the exploration. language_code_to_contents_mapping.update( @@ -1259,23 +1579,22 @@ def regenerate_voiceovers_on_exploration_added_to_topic( entity_translations ) ) + logging.info( + 'Voiceover regeneration logs: %s' % language_code_to_contents_mapping + ) - _regenerate_voiceovers_for_given_contents( + regenerate_voiceovers_for_given_contents( exploration_id, - exploration_title, exploration_version, language_code_to_contents_mapping, - date_time, - author_id, - task_run_id=task_run_id, + task_run_id, ) def regenerate_voiceovers_of_exploration_for_given_language_accent( exploration_id: str, language_accent_code: str, - author_id: str, - date_time: str, + cloud_task_run_id: str, ) -> None: """Regenerates voiceovers of the provided exploration for the given language accent code. @@ -1288,14 +1607,20 @@ def regenerate_voiceovers_of_exploration_for_given_language_accent( need to be regenerated. language_accent_code: str. The language accent code for which voiceovers need to be regenerated. - author_id: str. The ID of the user who initiated the voiceover - regeneration. - date_time: str. The timestamp when the voiceover regeneration was - initiated. + cloud_task_run_id: str. The unique identifier for the voiceover + regeneration task. Raises: Exception. If the provided language accent code is invalid. """ + logging.info( + 'Voiceover regeneration logs: Started regenerating voiceovers for ' + 'exploration with ID: %s and language accent code: %s.' + % ( + exploration_id, + language_accent_code, + ) + ) # A dictionary where each key is a language code, and each value is a # content mapping dictionary. The content mapping dictionary contains # content IDs as keys and their corresponding HTML content as values. @@ -1314,9 +1639,8 @@ def regenerate_voiceovers_of_exploration_for_given_language_accent( assert exploration is not None exploration_version = exploration.version - exploration_title = exploration.title - if language_code == constants.DEFAULT_LANGUAGE_CODE: + if language_code == constants.constants.DEFAULT_LANGUAGE_CODE: # Retrieve all English-language contents from the exploration. language_code_to_contents_mapping.update( extract_english_voiceover_texts_from_exploration(exploration) @@ -1335,49 +1659,48 @@ def regenerate_voiceovers_of_exploration_for_given_language_accent( [entity_translation] ) ) - - _regenerate_voiceovers_for_given_contents( + logging.info( + 'Voiceover regeneration logs: %s' % language_code_to_contents_mapping + ) + regenerate_voiceovers_for_given_contents( exploration_id, - exploration_title, exploration_version, language_code_to_contents_mapping, - date_time, - author_id, + cloud_task_run_id, specific_language_accent_code=language_accent_code, ) -def generate_voiceover_from_translated_content( - exploration_id: str, - exploration_version: int, - translation_content: str, - content_id: str, - language_code: str, +def regenerate_voiceovers_after_accepting_suggestion( + suggestion_id: str, + task_run_id: str, ) -> None: - """Generates a new voiceover for translated content once translation - suggestions are approved by reviewers. + """Regenerates voiceover for the given content ID and language code after + accepting a translation suggestion. Args: - exploration_id: str. The ID of the exploration. - exploration_version: int. The version of the exploration. - translation_content: str. The translated content for which the - voiceover needs to be generated. - content_id: str. The content ID for which the voiceover is being - generated. - language_code: str. The language code for the voiceover. + suggestion_id: str. The ID of the suggestion. + task_run_id: str. The ID of the task run. """ + logging.info( + 'Voiceover regeneration logs: Started regenerating voiceovers after ' + 'accepting suggestion with ID: %s.' % suggestion_id, + ) + suggestion = suggestion_services.get_suggestion_by_id(suggestion_id) + translated_html_content = suggestion.change_cmd.translation_html + content_id = suggestion.change_cmd.content_id + language_code = suggestion.language_code + exploration_id = suggestion.target_id + exploration_version = suggestion.target_version_at_submission language_code_to_contents_mapping = { - language_code: {content_id: translation_content} + language_code: {content_id: translated_html_content} } - exploration = exp_fetchers.get_exploration_by_id(exploration_id) - assert exploration is not None - exploration_title = exploration.title - - _regenerate_voiceovers_for_given_contents( + logging.info( + 'Voiceover regeneration logs: %s' % language_code_to_contents_mapping + ) + regenerate_voiceovers_for_given_contents( exploration_id, - exploration_title, exploration_version, language_code_to_contents_mapping, - datetime.datetime.utcnow().isoformat(), - feconf.SYSTEM_COMMITTER_ID, + task_run_id, ) diff --git a/core/domain/voiceover_services_test.py b/core/domain/voiceover_services_test.py index c4a81cc0b4917..b219489302975 100644 --- a/core/domain/voiceover_services_test.py +++ b/core/domain/voiceover_services_test.py @@ -18,9 +18,9 @@ from __future__ import annotations -import datetime import json import os +import uuid from core import feconf, schema_utils from core.constants import constants @@ -28,6 +28,8 @@ from core.domain import platform_parameter_list as param_list from core.domain import ( state_domain, + suggestion_services, + taskqueue_services, translation_domain, translation_fetchers, voiceover_domain, @@ -781,6 +783,67 @@ def test_should_successfully_get_autogeneratable_accents(self) -> None: ) self.assertItemsEqual(autogeneratable_accents_for_hindi, []) + def test_get_new_auto_voiceover_accent_returns_new_enabled_accent( + self, + ) -> None: + existing_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True}, + 'hi': {'hi-IN': False}, + } + voiceover_services.save_language_accent_support( + language_codes_mapping=existing_language_accent_mapping + ) + updated_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True, 'en-IN': True}, + 'hi': {'hi-IN': False}, + } + + new_accent_code = voiceover_services.get_new_auto_voiceover_accent( + updated_language_accent_mapping + ) + + self.assertEqual(new_accent_code, 'en-IN') + + def test_get_new_auto_voiceover_accent_returns_enabled_existing_accent( + self, + ) -> None: + existing_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True}, + 'hi': {'hi-IN': False}, + } + voiceover_services.save_language_accent_support( + language_codes_mapping=existing_language_accent_mapping + ) + updated_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True}, + 'hi': {'hi-IN': True}, + } + + new_accent_code = voiceover_services.get_new_auto_voiceover_accent( + updated_language_accent_mapping + ) + + self.assertEqual(new_accent_code, 'hi-IN') + + def test_get_new_auto_voiceover_accent_returns_none_when_no_new_enabled_accent( + self, + ) -> None: + existing_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True}, + } + voiceover_services.save_language_accent_support( + language_codes_mapping=existing_language_accent_mapping + ) + updated_language_accent_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True, 'en-IN': False}, + } + + new_accent_code = voiceover_services.get_new_auto_voiceover_accent( + updated_language_accent_mapping + ) + + self.assertIsNone(new_accent_code) + class VoiceoversLanguageAccentConstantsTests(test_utils.GenericTestBase): """Unit tests to validate the language-accent information saved as @@ -788,9 +851,8 @@ class VoiceoversLanguageAccentConstantsTests(test_utils.GenericTestBase): """ def test_get_language_accent_master_list_works_correctly(self) -> None: - file_path = os.path.join( - feconf.VOICEOVERS_DATA_DIR, 'language_accent_master_list.json' - ) + file_path = os.path.join('assets', 'language_accent_master_list.json') + with open(file_path, 'r', encoding='utf-8') as f: language_accent_master_list: Dict[str, Dict[str, str]] = json.loads( f.read() @@ -808,7 +870,7 @@ def test_get_autogeneratable_language_accent_list_works_correctly( self, ) -> None: file_path = os.path.join( - feconf.VOICEOVERS_DATA_DIR, + 'assets', 'autogeneratable_language_accent_list.json', ) with open(file_path, 'r', encoding='utf-8') as f: @@ -933,6 +995,31 @@ def test_should_get_correct_language_code_for_given_accent(self) -> None: expected_language_code, ) + def test_validate_language_accent_code_for_autogeneration(self) -> None: + invalid_accent_code = 'en-XX' + self.assertFalse( + voiceover_services.is_accent_code_valid_for_autogeneration( + invalid_accent_code + ) + ) + + # Here we use MyPy ignore because here we assign type int to + # type str. This is done to test the validation of the + # is_accent_code_valid_for_autogeneration method. + invalid_accent_code = 5 # type: ignore[assignment] + self.assertFalse( + voiceover_services.is_accent_code_valid_for_autogeneration( + invalid_accent_code + ) + ) + + valid_accent_code = 'en-US' + self.assertTrue( + voiceover_services.is_accent_code_valid_for_autogeneration( + valid_accent_code + ) + ) + class VoiceoverRegenerationTests(test_utils.GenericTestBase): """Test class to verify voiceover regeneration across various scenarios, @@ -987,12 +1074,8 @@ def test_should_regenerate_voiceover_for_curated_exploration_content_update( self, ) -> None: exploration_id = 'exp_id_1' - exploration_title = 'Test Exploration' exploration_version = 2 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - - date_time = datetime.datetime.utcnow().isoformat() commit1 = exp_models.ExplorationCommitLogEntryModel.create( exploration_id, @@ -1030,18 +1113,34 @@ def test_should_regenerate_voiceover_for_curated_exploration_content_update( ) self.assertEqual(len(entity_voiceovers_models), 0) - with self.swap( - voiceover_services, - 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', - self.mock_send_email_to_voiceover_admins_and_tech_leads, - ): - voiceover_services.regenerate_voiceovers_on_exploration_update( - exploration_id=exploration_id, - exploration_title=exploration_title, - exploration_version=exploration_version, - author_id=author_id, - date_time=date_time, - ) + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + voiceover_services.regenerate_voiceovers_on_exploration_update( + exploration_id, exploration_version, parent_cloud_task_model_id + ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) entity_voiceovers_models = ( voiceover_services.get_entity_voiceovers_for_given_exploration( @@ -1059,11 +1158,8 @@ def test_should_regenerate_voiceover_for_curated_exploration_content_update( def test_should_regenerate_voiceover_for_translation_update(self) -> None: exploration_id = 'exp_id_1' - exploration_title = 'Test Exploration' exploration_version = 2 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() commit1 = exp_models.ExplorationCommitLogEntryModel.create( exploration_id, @@ -1098,18 +1194,34 @@ def test_should_regenerate_voiceover_for_translation_update(self) -> None: ) self.assertEqual(len(entity_voiceovers_models), 0) - with self.swap( - voiceover_services, - 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', - self.mock_send_email_to_voiceover_admins_and_tech_leads, - ): - voiceover_services.regenerate_voiceovers_on_exploration_update( - exploration_id=exploration_id, - exploration_title=exploration_title, - exploration_version=exploration_version, - author_id=author_id, - date_time=date_time, - ) + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + voiceover_services.regenerate_voiceovers_on_exploration_update( + exploration_id, exploration_version, parent_cloud_task_model_id + ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) entity_voiceovers_models = ( voiceover_services.get_entity_voiceovers_for_given_exploration( @@ -1125,10 +1237,7 @@ def test_should_raise_exception_when_change_diff_is_not_accessible( self, ) -> None: exploration_id = 'exp_id_1' - exploration_title = 'Test Exploration' exploration_version = 2 - author_id = 'nik' - date_time = datetime.datetime.utcnow().isoformat() error = ( 'Could not fetch change diff for exploration %s, version %s during ' @@ -1136,24 +1245,29 @@ def test_should_raise_exception_when_change_diff_is_not_accessible( % (exploration_id, str(exploration_version)) ) + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + with self.assertRaisesRegex(Exception, error): voiceover_services.regenerate_voiceovers_on_exploration_update( - exploration_id=exploration_id, - exploration_title=exploration_title, - exploration_version=exploration_version, - author_id=author_id, - date_time=date_time, + exploration_id, exploration_version, parent_cloud_task_model_id ) def test_should_not_regenerate_voiceover_for_non_supported_accents( self, ) -> None: exploration_id = 'exp_id_1' - exploration_title = 'Test Exploration' exploration_version = 2 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() commit1 = exp_models.ExplorationCommitLogEntryModel.create( exploration_id, @@ -1195,18 +1309,34 @@ def test_should_not_regenerate_voiceover_for_non_supported_accents( ) self.assertEqual(len(entity_voiceovers_models), 0) - with self.swap( - voiceover_services, - 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', - self.mock_send_email_to_voiceover_admins_and_tech_leads, - ): - voiceover_services.regenerate_voiceovers_on_exploration_update( - exploration_id=exploration_id, - exploration_title=exploration_title, - exploration_version=exploration_version, - author_id=author_id, - date_time=date_time, - ) + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + voiceover_services.regenerate_voiceovers_on_exploration_update( + exploration_id, exploration_version, parent_cloud_task_model_id + ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) entity_voiceovers_models = ( voiceover_services.get_entity_voiceovers_for_given_exploration( @@ -1297,25 +1427,7 @@ def test_should_send_emails_to_voiceover_admins_and_tech_leads( ) def test_should_raise_error_while_regenerating_voiceover(self) -> None: exploration_id = 'exp_id_1' - exploration_title = 'Test Exploration' exploration_version = 2 - self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = '2025-08-01T08:35:05.864077' - cloud_task_run_model_id = ( - cloud_task_models.CloudTaskRunModel.get_new_id() - ) - cloud_task_models.CloudTaskRunModel.create_cloud_task_run_model( - cloud_task_run_model_id=cloud_task_run_model_id, - cloud_task_name=( - 'projects/dev-project-id/locations/us-central1/queues/' - 'voiceover-regeneration/tasks/task1' - ), - latest_job_state='RUNNING', - function_id='update_stats', - current_retry_attempt=1, - ) - commit1 = exp_models.ExplorationCommitLogEntryModel.create( exploration_id, 2, @@ -1363,28 +1475,52 @@ def mock_regenerate_voiceovers_of_exploration( _language_accent_code: str, ) -> List[Tuple[str, str]]: errors_while_voiceover_regeneration = [ - ('content5', 'Error 1 occurred'), + ('content_5', 'Error 1 occurred'), ] return errors_while_voiceover_regeneration + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + + voiceover_services.regenerate_voiceovers_on_exploration_update( + exploration_id, exploration_version, parent_cloud_task_model_id + ) + + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + with self.swap( voiceover_regeneration_services, 'regenerate_voiceovers_of_exploration', mock_regenerate_voiceovers_of_exploration, ): - voiceover_services.regenerate_voiceovers_on_exploration_update( - exploration_id=exploration_id, - exploration_title=exploration_title, - exploration_version=exploration_version, - author_id=author_id, - date_time=date_time, - task_run_id=cloud_task_run_model_id, - ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) all_models: Sequence[email_models.SentEmailModel] = ( email_models.SentEmailModel.get_all().fetch() ) - self.assertEqual(len(all_models), 3) + + self.assertEqual(len(all_models), 2) expected_html_body = ( 'Hi Voiceover Admins,tester has initiated the generation ' @@ -1407,13 +1543,104 @@ def mock_regenerate_voiceovers_of_exploration( self.assertEqual(email_model.html_body, expected_html_body) updated_cloud_task_run_model = cloud_task_models.CloudTaskRunModel.get( - cloud_task_run_model_id + parent_cloud_task_model_id ) assert updated_cloud_task_run_model is not None self.assertEqual( updated_cloud_task_run_model.latest_job_state, 'PERMANENTLY_FAILED' ) + def test_should_raise_exception_while_regenerating_voiceovers_in_batch( + self, + ) -> None: + exploration_id = 'exp_id_1' + exploration_version = 2 + commit1 = exp_models.ExplorationCommitLogEntryModel.create( + exploration_id, + 2, + self.committer_1_id, + 'msg', + 'create', + [ + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'State 1', + 'old_value': { + 'content_id': 'content_5', + 'html': self.old_content_html, + }, + 'new_value': { + 'content_id': 'content_5', + 'html': self.new_content_html, + }, + } + ], + constants.ACTIVITY_STATUS_PRIVATE, + False, + ) + + commit1.exploration_id = exploration_id + commit1.update_timestamps() + commit1.put() + + self.voiceover_autogeneration_policy_model.language_codes_mapping = { + 'en': {'en-US': True} + } + + entity_voiceovers_models = ( + voiceover_services.get_entity_voiceovers_for_given_exploration( + exploration_id, 'exploration', exploration_version + ) + ) + self.assertEqual(len(entity_voiceovers_models), 0) + + def mock_regenerate_voiceovers_of_exploration( + _exploration_id: str, + _exploration_version: int, + _content_id_to_content_html: Dict[str, str], + _language_accent_code: str, + ) -> List[Tuple[str, str]]: + raise Exception( + 'Expected error raised during voiceover regeneration!' + ) + + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + + voiceover_services.regenerate_voiceovers_on_exploration_update( + exploration_id, exploration_version, parent_cloud_task_model_id + ) + + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + with self.swap( + voiceover_regeneration_services, + 'regenerate_voiceovers_of_exploration', + mock_regenerate_voiceovers_of_exploration, + ): + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) + def _create_exploration_and_arabic_translation( self, exploration_id: str, language_code: str ) -> None: @@ -1423,6 +1650,9 @@ def _create_exploration_and_arabic_translation( exploration_id: str. The ID of the exploration to create. language_code: str. The language code for the translation. """ + self.signup(self.OWNER_EMAIL, self.OWNER_USERNAME) + owner_id = self.get_user_id_from_email(self.OWNER_EMAIL) + exploration = exp_domain.Exploration.create_default_exploration( exploration_id, title='A Title', @@ -1470,7 +1700,7 @@ def _create_exploration_and_arabic_translation( ] exploration.add_state('Second', 'content_2', 'content-3') - exp_services.save_new_exploration(exploration_id, exploration) + exp_services.save_new_exploration(owner_id, exploration) arabic_translation = 'المحتوى المترجم' translations_mapping: Dict[str, feconf.TranslatedContentDict] = { @@ -1510,8 +1740,6 @@ def test_should_regenerate_voiceover_for_arabic_language(self) -> None: exploration_id = 'exp_id_1' exploration_version = 1 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() self._create_exploration_and_arabic_translation( exploration_id, language_code @@ -1535,16 +1763,34 @@ def test_should_regenerate_voiceover_for_arabic_language(self) -> None: self.assertEqual(entity_translation.language_code, language_code) self.assertEqual(entity_voiceovers.voiceovers_mapping, {}) - with self.swap( - voiceover_services, - 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', - self.mock_send_email_to_voiceover_admins_and_tech_leads, - ): - ( - voiceover_services.regenerate_voiceovers_of_exploration_for_given_language_accent( - exploration_id, language_accent_code, author_id, date_time + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + voiceover_services.regenerate_voiceovers_of_exploration_for_given_language_accent( + exploration_id, language_accent_code, parent_cloud_task_model_id + ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, ) - ) entity_voiceovers = ( voiceover_services.get_voiceovers_for_given_language_accent_code( @@ -1585,15 +1831,13 @@ def test_should_raise_exception_when_language_accent_code_is_not_supported( language_accent_code = 'ar-non-existent' exploration_id = 'exp_id_1' self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() with self.assertRaisesRegex( Exception, 'Invalid language accent code: %s' % language_accent_code ): ( voiceover_services.regenerate_voiceovers_of_exploration_for_given_language_accent( - exploration_id, language_accent_code, author_id, date_time + exploration_id, language_accent_code, 'cloud_task_run_id' ) ) @@ -1603,8 +1847,6 @@ def test_should_regenerate_voiceover_for_english_language(self) -> None: exploration_id = 'exp_id_1' exploration_version = 1 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() self._create_exploration_and_arabic_translation( exploration_id, language_code @@ -1620,16 +1862,34 @@ def test_should_regenerate_voiceover_for_english_language(self) -> None: ) self.assertEqual(entity_voiceovers.voiceovers_mapping, {}) - with self.swap( - voiceover_services, - 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', - self.mock_send_email_to_voiceover_admins_and_tech_leads, - ): - ( - voiceover_services.regenerate_voiceovers_of_exploration_for_given_language_accent( - exploration_id, language_accent_code, author_id, date_time + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + voiceover_services.regenerate_voiceovers_of_exploration_for_given_language_accent( + exploration_id, language_accent_code, parent_cloud_task_model_id + ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, ) - ) entity_voiceovers = ( voiceover_services.get_voiceovers_for_given_language_accent_code( @@ -1644,23 +1904,22 @@ def test_should_regenerate_voiceover_for_english_language(self) -> None: entity_voiceovers.language_accent_code, language_accent_code ) self.assertNotEqual(entity_voiceovers.voiceovers_mapping, {}) - self.assertEqual( + default_audio_offset = [ + {'token': 'This', 'audio_offset_msecs': 0.0}, + {'token': 'is', 'audio_offset_msecs': 100.0}, + {'token': 'a', 'audio_offset_msecs': 200.0}, + {'token': 'test', 'audio_offset_msecs': 300.0}, + {'token': 'text', 'audio_offset_msecs': 400.0}, + ] + self.assertDictEqual( entity_voiceovers.automated_voiceovers_audio_offsets_msecs, { - 'content_0': [ - {'token': 'This', 'audio_offset_msecs': 0.0}, - {'token': 'is', 'audio_offset_msecs': 100.0}, - {'token': 'a', 'audio_offset_msecs': 200.0}, - {'token': 'test', 'audio_offset_msecs': 300.0}, - {'token': 'text', 'audio_offset_msecs': 400.0}, - ], - 'feedback_1': [ - {'token': 'This', 'audio_offset_msecs': 0.0}, - {'token': 'is', 'audio_offset_msecs': 100.0}, - {'token': 'a', 'audio_offset_msecs': 200.0}, - {'token': 'test', 'audio_offset_msecs': 300.0}, - {'token': 'text', 'audio_offset_msecs': 400.0}, - ], + 'content_0': default_audio_offset, + 'feedback_1': default_audio_offset, + 'content_2': default_audio_offset, + 'default_outcome_1': default_audio_offset, + 'content-3': default_audio_offset, + 'ca_placeholder_2': default_audio_offset, }, ) @@ -1671,8 +1930,6 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( exploration_id = 'exp_id_1' exploration_version = 1 self.signup('tester@org.com', 'tester') - author_id = self.get_user_id_from_email('tester@org.com') - date_time = datetime.datetime.utcnow().isoformat() self._create_exploration_and_arabic_translation( exploration_id, language_code @@ -1686,11 +1943,11 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( ) self.assertEqual(len(entity_voiceovers_list), 0) - cloud_task_run_model_id = ( + parent_cloud_task_run_model_id = ( cloud_task_models.CloudTaskRunModel.get_new_id() ) cloud_task_models.CloudTaskRunModel.create_cloud_task_run_model( - cloud_task_run_model_id=cloud_task_run_model_id, + cloud_task_run_model_id=parent_cloud_task_run_model_id, cloud_task_name=( 'projects/dev-project-id/locations/us-central1/queues/' 'voiceover-regeneration/tasks/task1' @@ -1702,10 +1959,10 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( voiceover_regeneration_task_mapping_model_id = '%s:%s' % ( exploration_id, - cloud_task_run_model_id, + parent_cloud_task_run_model_id, ) voiceover_regeneration_task_mapping_model = ( - cloud_task_models.VoiceoverRegenerationTaskMappingModel.get( + cloud_task_models.VoiceoverRegenerationJobModel.get( voiceover_regeneration_task_mapping_model_id, strict=False ) ) @@ -1720,11 +1977,23 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( ( voiceover_services.regenerate_voiceovers_on_exploration_added_to_topic( exploration_id, - date_time, - author_id, - cloud_task_run_model_id, + parent_cloud_task_run_model_id, ) ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_run_model_id, + cloud_run.task_run_id, + ) entity_voiceovers_list = ( voiceover_services.get_entity_voiceovers_for_given_exploration( @@ -1739,13 +2008,27 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( self.assertEqual(len(entity_voiceovers_list), 3) voiceover_regeneration_task_mapping_model = ( - cloud_task_models.VoiceoverRegenerationTaskMappingModel.get( + cloud_task_models.VoiceoverRegenerationJobModel.get( voiceover_regeneration_task_mapping_model_id, strict=False ) ) expected_language_accent_to_content_status_map = { - 'en-US': {'content_0': 'SUCCEEDED', 'feedback_1': 'SUCCEEDED'}, - 'en-IN': {'content_0': 'SUCCEEDED', 'feedback_1': 'SUCCEEDED'}, + 'en-US': { + 'ca_placeholder_2': 'SUCCEEDED', + 'content-3': 'SUCCEEDED', + 'content_0': 'SUCCEEDED', + 'content_2': 'SUCCEEDED', + 'default_outcome_1': 'SUCCEEDED', + 'feedback_1': 'SUCCEEDED', + }, + 'en-IN': { + 'ca_placeholder_2': 'SUCCEEDED', + 'content-3': 'SUCCEEDED', + 'content_0': 'SUCCEEDED', + 'content_2': 'SUCCEEDED', + 'default_outcome_1': 'SUCCEEDED', + 'feedback_1': 'SUCCEEDED', + }, 'ar-AE': {'content_0': 'SUCCEEDED', 'feedback_1': 'SUCCEEDED'}, } @@ -1764,16 +2047,24 @@ def test_should_regenerate_voiceover_when_exploration_is_curated( ) self.assertEqual( voiceover_regeneration_task_mapping_model.cloud_task_run_id, - cloud_task_run_model_id, + parent_cloud_task_run_model_id, ) def test_should_generate_voiceover_for_translated_content(self) -> None: - language_code = 'ar' + language_code = 'hi' exploration_id = 'exp_id_1' + language_codes_mapping: Dict[str, Dict[str, bool]] = { + 'en': {'en-US': True}, + 'hi': {'hi-IN': True}, + } + voiceover_services.save_language_accent_support( + language_codes_mapping=language_codes_mapping + ) exploration_version = 1 - language_accent_code = 'ar-AE' - translation_content = 'المحتوى المترجم' - content_id = 'content_id_0' + language_accent_code = 'hi-IN' + + self.signup(self.OWNER_EMAIL, self.OWNER_USERNAME) + owner_id = self.get_user_id_from_email(self.OWNER_EMAIL) exploration = exp_domain.Exploration.create_default_exploration( exploration_id, @@ -1782,7 +2073,27 @@ def test_should_generate_voiceover_for_translated_content(self) -> None: objective='An Objective', ) exploration.states['Introduction'].content.html = 'First Card!' - exp_services.save_new_exploration(exploration_id, exploration) + exp_services.save_new_exploration(owner_id, exploration) + + add_translation_change_dict = { + 'cmd': exp_domain.CMD_ADD_WRITTEN_TRANSLATION, + 'state_name': 'Introduction', + 'content_id': 'content_0', + 'language_code': language_code, + 'content_html': 'First Card!', + 'translation_html': 'पहला कार्ड', + 'data_format': 'html', + } + + translation_suggestion = suggestion_services.create_suggestion( + feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT, + feconf.ENTITY_TYPE_EXPLORATION, + exploration_id, + exploration.version, + owner_id, + add_translation_change_dict, + 'test description', + ) entity_voiceovers = ( voiceover_services.get_voiceovers_for_given_language_accent_code( @@ -1794,21 +2105,45 @@ def test_should_generate_voiceover_for_translated_content(self) -> None: ) self.assertEqual(entity_voiceovers.voiceovers_mapping, {}) + parent_cloud_task_model_id = 'cloud_task_model_id' + task_name = 'projects/%s/locations/%s/queues/%s/tasks/%s' % ( + 'dev-project-id', + 'us-central', + 'voiceover-regeneration', + uuid.uuid4().hex, + ) + function_id = 'regenerate_voiceovers_on_exploration_update' + taskqueue_services.create_new_cloud_task_model( + parent_cloud_task_model_id, task_name, function_id + ) + with self.swap( voiceover_services, 'send_email_to_voiceover_admins_and_tech_leads_after_regeneration', self.mock_send_email_to_voiceover_admins_and_tech_leads, ): ( - voiceover_services.generate_voiceover_from_translated_content( - exploration_id, - exploration_version, - translation_content, - content_id, - language_code, + voiceover_services.regenerate_voiceovers_after_accepting_suggestion( + translation_suggestion.suggestion_id, + parent_cloud_task_model_id, ) ) + updated_cloud_task_runs = sorted( + taskqueue_services.get_all_cloud_task_runs(), + key=lambda task_run: task_run.created_on, + ) + for cloud_run in updated_cloud_task_runs: + if ( + cloud_run.function_id + == 'regenerate_voiceovers_for_batch_contents' + ): + voiceover_services.regenerate_voiceovers_for_batch_contents( + exploration_id, + parent_cloud_task_model_id, + cloud_run.task_run_id, + ) + entity_voiceovers = ( voiceover_services.get_voiceovers_for_given_language_accent_code( feconf.ENTITY_TYPE_EXPLORATION, @@ -1817,4 +2152,3 @@ def test_should_generate_voiceover_for_translated_content(self) -> None: language_accent_code, ) ) - self.assertNotEqual(entity_voiceovers.voiceovers_mapping, {}) diff --git a/core/feature_flag_list.py b/core/feature_flag_list.py index 0d34afcc8ca24..5baedfed0575f 100644 --- a/core/feature_flag_list.py +++ b/core/feature_flag_list.py @@ -69,6 +69,9 @@ class FeatureNames(enum.Enum): SHOW_VOICEOVER_TAB_FOR_NON_CURATED_EXPLORATIONS = ( 'show_voiceover_tab_for_non_curated_explorations' ) + HIGHLIGHT_SENTENCES_DURING_AUTOMATIC_VOICEOVER_PLAYBACK = ( + 'highlight_sentences_during_automatic_voiceover_playback' + ) SHOW_RESTRUCTURED_STUDY_GUIDES = 'show_restructured_study_guides' ENABLE_TRANSLATION_OPPORTUNITIES_WITH_NEW_OPP_MODELS = ( 'enable_translation_opps_with_new_opp_models' @@ -83,6 +86,19 @@ class FeatureNames(enum.Enum): 'enable_background_voiceover_synthesis' ) ENABLE_READY_FOR_REVIEW_TEST = 'enable_ready_for_review_test' + ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER = ( + 'enable_financial_literacy_campaign_banner' + ) + # A separate flag is used for testing the financial literacy campaign banner with early dates. + # This allows testing the feature before the actual campaign dates that will + # be used in production. Without a separate test flag, we would need to change + # the campaign date values for testing and then update them again before + # releasing to production. That process would require additional PRs, + # cherry-picks, or hotfixes. Using a dedicated test-mode flag avoids that + # overhead and keeps testing and production configurations separate. + ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER_TEST_MODE = ( + 'enable_financial_literacy_campaign_banner_test_mode' + ) # Names of feature objects defined in FeatureNames should be added @@ -123,8 +139,10 @@ class FeatureNames(enum.Enum): FeatureNames.SHOW_VOICEOVER_TAB_FOR_NON_CURATED_EXPLORATIONS, FeatureNames.NEW_LESSON_PLAYER, FeatureNames.AUTOMATIC_VOICEOVER_REGENERATION_FROM_EXP, + FeatureNames.HIGHLIGHT_SENTENCES_DURING_AUTOMATIC_VOICEOVER_PLAYBACK, FeatureNames.SHOW_REGENERATED_VOICEOVERS_TO_LEARNERS, FeatureNames.ENABLE_BACKGROUND_VOICEOVER_SYNTHESIS, + FeatureNames.ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER_TEST_MODE, ] # Names of features in prod stage, the corresponding feature flag instances must @@ -136,8 +154,8 @@ class FeatureNames(enum.Enum): FeatureNames.EXPLORATION_EDITOR_CAN_MODIFY_TRANSLATIONS, FeatureNames.EXPLORATION_EDITOR_CAN_TAG_MISCONCEPTIONS, FeatureNames.SHOW_REDESIGNED_LEARNER_DASHBOARD, - FeatureNames.ENABLE_WORKED_EXAMPLES_RTE_COMPONENT, FeatureNames.SHOW_RESTRUCTURED_STUDY_GUIDES, + FeatureNames.ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER, ] # Names of features that should not be used anymore, e.g. features that are @@ -154,6 +172,7 @@ class FeatureNames(enum.Enum): FeatureNames.AUTO_UPDATE_EXP_VOICE_ARTIST_LINK, FeatureNames.LABEL_ACCENT_TO_VOICE_ARTIST, FeatureNames.ADD_VOICEOVER_WITH_ACCENT, + FeatureNames.ENABLE_WORKED_EXAMPLES_RTE_COMPONENT, ] FEATURE_FLAG_NAME_TO_DESCRIPTION_AND_FEATURE_STAGE = { @@ -285,13 +304,6 @@ class FeatureNames(enum.Enum): feature_flag_domain.ServerMode.DEV, ) ), - FeatureNames.ENABLE_WORKED_EXAMPLES_RTE_COMPONENT.value: ( - ( - 'Allows creators to add worked examples to the review material ' - 'section of skills and explanation of the study guides.', - feature_flag_domain.ServerMode.PROD, - ) - ), FeatureNames.SHOW_REGENERATED_VOICEOVERS_TO_LEARNERS.value: ( ( 'This flag allows learners to see the regenerated voiceovers ' @@ -299,6 +311,14 @@ class FeatureNames(enum.Enum): feature_flag_domain.ServerMode.TEST, ) ), + FeatureNames.HIGHLIGHT_SENTENCES_DURING_AUTOMATIC_VOICEOVER_PLAYBACK.value: ( + ( + 'This flag enables the highlighting of sentences during the ' + 'automatic voiceover playback in the exploration player and ' + 'editor pages.', + feature_flag_domain.ServerMode.TEST, + ) + ), FeatureNames.ENABLE_BACKGROUND_VOICEOVER_SYNTHESIS.value: ( ( 'The flag enables the asynchronous voiceover synthesis for the ' @@ -312,4 +332,16 @@ class FeatureNames(enum.Enum): feature_flag_domain.ServerMode.DEV, ) ), + FeatureNames.ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER.value: ( + ( + 'This flag enables the financial literacy campaign banner for the fundraising campaign.', + feature_flag_domain.ServerMode.PROD, + ) + ), + FeatureNames.ENABLE_FINANCIAL_LITERACY_CAMPAIGN_BANNER_TEST_MODE.value: ( + ( + 'This flag enables the financial literacy campaign banner for the fundraising campaign in test mode.', + feature_flag_domain.ServerMode.TEST, + ) + ), } diff --git a/core/feconf.py b/core/feconf.py index 5f4dea17f7fcd..c5fc0e0e3a198 100644 --- a/core/feconf.py +++ b/core/feconf.py @@ -84,7 +84,6 @@ def check_dev_mode_is_true() -> None: SAMPLE_EXPLORATIONS_DIR = os.path.join('data', 'explorations') SAMPLE_COLLECTIONS_DIR = os.path.join('data', 'collections') CONTENT_VALIDATION_DIR = os.path.join('core', 'domain') -VOICEOVERS_DATA_DIR = os.path.join('data', 'voiceovers') SAMPLE_AUTO_VOICEOVERS_DATA_DIR = os.path.join( 'assets', 'sample-autogenerated-voiceovers-for-dev' ) @@ -543,7 +542,7 @@ def get_empty_ratings() -> Dict[str, int]: DATAFLOW_TEMP_LOCATION_TEMPLATE = 'gs://%s-beam-jobs-temp/' DATAFLOW_STAGING_LOCATION_TEMPLATE = 'gs://%s-beam-jobs-staging/' -OPPIA_VERSION = '3.5.0' +OPPIA_VERSION = '3.5.1' OPPIA_PYTHON_PACKAGE_PATH = './build/oppia_beam_job-%s.tar.gz' % OPPIA_VERSION # Committer id for system actions. The username for the system committer @@ -852,6 +851,9 @@ def get_empty_ratings() -> Dict[str, int]: '%s/email/contributordashboardachievementnotificationemailhandler' % (TASKQUEUE_URL_PREFIX) ) +TASK_URL_RETRY_FAILED_EMAIL = ( + '%s/email/retryemailhandler' % TASKQUEUE_URL_PREFIX +) TASK_URL_DEFERRED = '%s/deferredtaskshandler' % TASKQUEUE_URL_PREFIX # TODO(sll): Add all other URLs here. @@ -1077,8 +1079,7 @@ def get_empty_ratings() -> Dict[str, int]: '/regenerate_automatic_voiceover/
This is the first card of first exploration.
', + }, + } + ), + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'End State', + 'new_value': { + 'content_id': 'content_3', + 'html': 'This is the last card of first exploration.
', + }, + } + ), + ], + 'Changes content.', + ) + + story_services.update_story( + self.owner_id, + self.STORY_ID_1, + [ + story_domain.StoryChange( + { + 'cmd': 'add_story_node', + 'node_id': 'node_1', + 'title': 'Node1', + } + ), + story_domain.StoryChange( + { + 'cmd': 'update_story_node_property', + 'property_name': 'exploration_id', + 'node_id': 'node_1', + 'old_value': None, + 'new_value': self.CURATED_EXPLORATION_ID_1, + } + ), + ], + 'Changes.', + ) + + exploration_2 = self.save_new_valid_exploration( + self.CURATED_EXPLORATION_ID_2, + self.owner_id, + title='title2', + category=constants.constants.ALL_CATEGORIES[0], + end_state_name='End State', + ) + self.publish_exploration(self.owner_id, exploration_2.id) + + exp_services.update_exploration( + self.owner_id, + self.CURATED_EXPLORATION_ID_2, + [ + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'Introduction', + 'new_value': { + 'content_id': 'content_0', + 'html': 'This is the first card of second exploration.
', + }, + } + ), + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'End State', + 'new_value': { + 'content_id': 'content_3', + 'html': 'This is the last card of second exploration.
', + }, + } + ), + ], + 'Changes content.', + ) + + story_services.update_story( + self.owner_id, + self.STORY_ID_2, + [ + story_domain.StoryChange( + { + 'cmd': 'add_story_node', + 'node_id': 'node_1', + 'title': 'Node1', + } + ), + story_domain.StoryChange( + { + 'cmd': 'update_story_node_property', + 'property_name': 'exploration_id', + 'node_id': 'node_1', + 'old_value': None, + 'new_value': self.CURATED_EXPLORATION_ID_2, + } + ), + ], + 'Changes.', + ) + + # Create a non-curated exploration. + exploration_3 = self.save_new_valid_exploration( + self.NON_CURATED_EXPLORATION_ID, + self.owner_id, + title='title3', + category=constants.constants.ALL_CATEGORIES[0], + end_state_name='End State', + ) + self.publish_exploration(self.owner_id, exploration_3.id) + + # Adding Hindi and Portuguese translations to the first and second. + translation_models.EntityTranslationsModel.create_new( + 'exploration', + self.CURATED_EXPLORATION_ID_1, + 2, + 'hi', + { + 'content_0': { + 'content_value': 'यह प्रथम अन्वेषण का पहला कार्ड है.
', + 'content_format': 'html', + 'needs_update': False, + }, + 'content_3': { + 'content_value': 'यह प्रथम अन्वेषण का अंतिम कार्ड है.
', + 'content_format': 'html', + 'needs_update': True, + }, + }, + ).put() + + translation_models.EntityTranslationsModel.create_new( + 'exploration', + self.CURATED_EXPLORATION_ID_2, + 2, + 'pt', + { + 'content_0': { + 'content_value': 'Esta é a primeira carta da segunda exploração.
', + 'content_format': 'html', + 'needs_update': False, + }, + 'content_3': { + 'content_value': 'Esta é a segunda carta da segunda exploração.
', + 'content_format': 'html', + 'needs_update': True, + }, + }, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_1, + 2, + 'hi-IN', + { + 'content_0': { + 'manual': None, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_2, + 2, + 'en-US', + { + 'content_0': { + 'manual': self.voiceover_dict_1, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_2, + 2, + 'pt-BR', + { + 'content_0': { + 'manual': None, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_autogeneration_policy_model = ( + voiceover_models.VoiceoverAutogenerationPolicyModel( + id=voiceover_models.VOICEOVER_AUTOGENERATION_POLICY_ID + ) + ) + voiceover_autogeneration_policy_model.language_codes_mapping = { + 'en': {'en-US': True, 'en-NG': False}, + 'hi': {'hi-IN': True}, + 'pt': {'pt-BR': True}, + 'ar': {'ar-AE': True}, + } + ( + voiceover_autogeneration_policy_model.autogenerated_voiceovers_are_enabled + ) = True + voiceover_autogeneration_policy_model.update_timestamps() + voiceover_autogeneration_policy_model.put() + + def _set_language_accent_code( + self, language_accent_code: Optional[str] + ) -> None: + """Sets the language accent code for the job. + + Args: + language_accent_code: str. The language accent code to set for the job. + """ + custom_options = self.pipeline.options.view_as(job_options.JobOptions) + custom_options.language_accent_code = language_accent_code + + +class VoiceoverSynthesisByAccentJobRunTests( + VoiceoverSynthesisByAccentBaseClass +): + """Tests for VoiceoverSynthesisByAccentJob.""" + + JOB_CLASS: Type[ + synthesize_voiceover_by_language_accent_jobs.VoiceoverSynthesisByAccentJob + ] = ( + synthesize_voiceover_by_language_accent_jobs.VoiceoverSynthesisByAccentJob + ) + + def test_empty_storage(self) -> None: + self._set_language_accent_code('en-US') + self.assert_job_output_is_empty() + + def test_should_regenerate_voiceover_successfully(self) -> None: + self._create_data_for_testing() + self._set_language_accent_code('en-US') + + expected_output_1 = ( + 'Exploration ID: exploration_id_1.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 101.\n' + ) + + expected_output_2 = ( + 'Exploration ID: exploration_id_2.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 103.\n' + ) + + expected_output = [ + job_run_result.JobRunResult(stdout=expected_output_1, stderr=''), + job_run_result.JobRunResult(stdout=expected_output_2, stderr=''), + ] + + self.assert_job_output_is(expected_output) + + def test_should_not_generate_voiceover_for_which_contents_are_not_available( + self, + ) -> None: + self._create_data_for_testing() + self._set_language_accent_code('ar-AE') + + expected_output_1 = ( + 'Exploration ID: exploration_id_1.\n' + 'No content found for language code: ar.' + ) + + expected_output_2 = ( + 'Exploration ID: exploration_id_2.\n' + 'No content found for language code: ar.' + ) + + expected_output = [ + job_run_result.JobRunResult(stdout=expected_output_1, stderr=''), + job_run_result.JobRunResult(stdout=expected_output_2, stderr=''), + ] + + self.assert_job_output_is(expected_output) + + def test_should_not_generate_voiceover_when_language_accent_code_is_none( + self, + ) -> None: + self._create_data_for_testing() + self._set_language_accent_code(None) + + expected_output_1 = 'Not generating voiceovers for exploration ID: exploration_id_1 since language accent code is None.' + + expected_output_2 = 'Not generating voiceovers for exploration ID: exploration_id_2 since language accent code is None.' + + expected_output = [ + job_run_result.JobRunResult(stdout=expected_output_1, stderr=''), + job_run_result.JobRunResult(stdout=expected_output_2, stderr=''), + ] + + self.assert_job_output_is(expected_output) + + def test_should_handle_failures_during_voiceover_regeneration(self) -> None: + def mock_synthesize_voiceover_for_html_string( + _exploration_id: str, + _content_html: str, + _language_accent_code: str, + _voiceover_filename: str, + _oppia_project_id: Optional[str], + ) -> List[Dict[str, Union[str, float]]]: + raise Exception('Failed to generate voiceovers.') + + self._create_data_for_testing() + self._set_language_accent_code('en-US') + + expected_output_1 = ( + 'Exploration ID: exploration_id_1.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Content IDs failed: [content_0, default_outcome_1, ca_placeholder_2, content_3]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + ) + expected_output_2 = ( + 'Exploration ID: exploration_id_2.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Content IDs failed: [content_0, default_outcome_1, ca_placeholder_2, content_3]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + ) + + with self.swap( + voiceover_regeneration_services, + 'synthesize_voiceover_for_html_string', + mock_synthesize_voiceover_for_html_string, + ): + expected_output = [ + job_run_result.JobRunResult( + stdout=expected_output_1, stderr='' + ), + job_run_result.JobRunResult( + stdout=expected_output_2, stderr='' + ), + ] + + self.assert_job_output_is(expected_output) + + def test_should_handle_empty_strings_during_voiceover_regeneration( + self, + ) -> None: + def mock_synthesize_voiceover_for_html_string( + _exploration_id: str, + _content_html: str, + _language_accent_code: str, + _voiceover_filename: str, + _oppia_project_id: Optional[str], + ) -> List[Dict[str, Union[str, float]]]: + return [] + + self._create_data_for_testing() + self._set_language_accent_code('en-US') + + expected_output_1 = ( + 'Exploration ID: exploration_id_1.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + ) + + expected_output_2 = ( + 'Exploration ID: exploration_id_2.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + ) + + with self.swap( + voiceover_regeneration_services, + 'synthesize_voiceover_for_html_string', + mock_synthesize_voiceover_for_html_string, + ): + expected_output = [ + job_run_result.JobRunResult( + stdout=expected_output_1, stderr='' + ), + job_run_result.JobRunResult( + stdout=expected_output_2, stderr='' + ), + ] + + self.assert_job_output_is(expected_output) diff --git a/core/jobs/batch_jobs/translation_audit_jobs.py b/core/jobs/batch_jobs/translation_audit_jobs.py new file mode 100644 index 0000000000000..34cf7b64fa516 --- /dev/null +++ b/core/jobs/batch_jobs/translation_audit_jobs.py @@ -0,0 +1,174 @@ +# coding: utf-8 +# +# Copyright 2026 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Beam jobs for auditing translation counts.""" + +from __future__ import annotations + +from core.jobs import base_jobs +from core.jobs.io import ndb_io +from core.jobs.types import job_run_result +from core.platform import models + +import apache_beam as beam +from typing import Dict, Iterable, Tuple + +MYPY = False +if MYPY: # pragma: no cover + from mypy_imports import opportunity_models, translation_models + +(opportunity_models, translation_models) = models.Registry.import_models( + [ + models.Names.OPPORTUNITY, + models.Names.TRANSLATION, + ] +) + + +class ValidateExplorationOpportunityCountsJob(base_jobs.JobBase): + """Job that validates translation_counts in + ExplorationOpportunitySummaryModel. + + This job computes the true translation counts by looking at + EntityTranslationsModel and compares it to the translation_counts + recorded in ExplorationOpportunitySummaryModel. It returns SUCCESS + if all counts match, and logs the mismatches otherwise. + """ + + def _get_translation_counts( + self, translation_model: translation_models.EntityTranslationsModel + ) -> Tuple[str, Tuple[str, int]]: + """Extracts translation counts from an EntityTranslationsModel. + + Args: + translation_model: EntityTranslationsModel. The model + to extract counts from. + + Returns: + tuple(str, tuple(str, int)). A tuple of + (exploration_id, (language_code, translation_count)). + """ + translation_count = len(translation_model.translations) + return ( + translation_model.entity_id, + (translation_model.language_code, translation_count), + ) + + def _validate_counts( + self, + exploration_id: str, + opportunity_summary_models_list: Iterable[ + opportunity_models.ExplorationOpportunitySummaryModel + ], + translation_counts_list: Iterable[Tuple[str, int]], + ) -> Iterable[job_run_result.JobRunResult]: + """Validates the translation counts for a given exploration. + + Args: + exploration_id: str. The exploration ID. + opportunity_summary_models_list: + list(ExplorationOpportunitySummaryModel). The list + of opportunity summary models for the exploration. + translation_counts_list: list(tuple(str, int)). True + counts from EntityTranslationsModel. + + Yields: + JobRunResult. Results detailing whether counts match or + describing the mismatches. + """ + summary_models = list(opportunity_summary_models_list) + actual_translations = list(translation_counts_list) + + if not summary_models: + return + + summary_model = summary_models[0] + stored_translation_counts = summary_model.translation_counts + + actual_translation_counts_dict: Dict[str, int] = {} + for language_code, count in actual_translations: + actual_translation_counts_dict[language_code] = count + + mismatch_found = False + + for lang_code, stored_count in stored_translation_counts.items(): + actual_count = actual_translation_counts_dict.get(lang_code, 0) + if stored_count != actual_count: + mismatch_found = True + yield job_run_result.JobRunResult.as_stderr( + 'Mismatch for exploration %s in %s: ' + 'stored=%s, actual=%s' + % (exploration_id, lang_code, stored_count, actual_count) + ) + + for lang_code, actual_count in actual_translation_counts_dict.items(): + if lang_code not in stored_translation_counts and actual_count > 0: + mismatch_found = True + yield job_run_result.JobRunResult.as_stderr( + 'Mismatch for exploration %s in %s: ' + 'stored=0 (missing), actual=%s' + % (exploration_id, lang_code, actual_count) + ) + + if not mismatch_found: + yield job_run_result.JobRunResult.as_stdout( + 'SUCCESS - Exploration %s counts are valid.' % exploration_id + ) + + def run(self) -> beam.PCollection[job_run_result.JobRunResult]: + """Returns a PCollection of results from the translation + count validation. + + Returns: + PCollection. A PCollection of JobRunResult objects. + """ + opportunity_summaries = ( + self.pipeline + | 'Get all ExplorationOpportunitySummaryModels' + >> ndb_io.GetModels( + opportunity_models.ExplorationOpportunitySummaryModel.get_all() + ) + | 'Key Opportunity by exploration_id' + >> beam.WithKeys( # pylint: disable=no-value-for-parameter + lambda model: model.id + ) + ) + + translation_counts = ( + self.pipeline + | 'Get all Exploration EntityTranslationsModels' + >> ndb_io.GetModels( + translation_models.EntityTranslationsModel.query( + translation_models.EntityTranslationsModel.entity_type + == 'exploration' + ) + ) + | 'Extract translation counts' + >> beam.Map(self._get_translation_counts) + ) + + grouped_data = { + 'opportunity_summary': opportunity_summaries, + 'translation_counts': translation_counts, + } | 'Group by exploration_id' >> beam.CoGroupByKey() + + return grouped_data | 'Process and Validate Counts' >> beam.FlatMap( + lambda kv: self._validate_counts( + exploration_id=kv[0], + opportunity_summary_models_list=(kv[1]['opportunity_summary']), + translation_counts_list=kv[1]['translation_counts'], + ) + ) diff --git a/core/jobs/batch_jobs/translation_audit_jobs_test.py b/core/jobs/batch_jobs/translation_audit_jobs_test.py new file mode 100644 index 0000000000000..7d591ca4e517d --- /dev/null +++ b/core/jobs/batch_jobs/translation_audit_jobs_test.py @@ -0,0 +1,200 @@ +# coding: utf-8 +# +# Copyright 2026 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for jobs.batch_jobs.translation_audit_jobs.""" + +from __future__ import annotations + +from core.jobs import job_test_utils +from core.jobs.batch_jobs import translation_audit_jobs +from core.jobs.types import job_run_result +from core.platform import models + +(opportunity_models, translation_models) = models.Registry.import_models( + [ + models.Names.OPPORTUNITY, + models.Names.TRANSLATION, + ] +) + + +class ValidateExplorationOpportunityCountsJobTests(job_test_utils.JobTestBase): + + JOB_CLASS = translation_audit_jobs.ValidateExplorationOpportunityCountsJob + + def test_empty_storage(self) -> None: + self.assert_job_output_is_empty() + + def test_matches_exactly(self) -> None: + exp_id = 'exp_1' + summary_model = opportunity_models.ExplorationOpportunitySummaryModel( + id=exp_id, + topic_id='topic1', + topic_name='Topic 1', + story_id='story_1', + story_title='Story 1', + chapter_title='Chapter 1', + content_count=10, + incomplete_translation_language_codes=['hi'], + translation_counts={'hi': 5, 'es': 10}, + language_codes_needing_voice_artists=[], + language_codes_with_assigned_voice_artists=[], + ) + summary_model.update_timestamps() + summary_model.put() + + translation_1 = translation_models.EntityTranslationsModel( + id='exploration-exp_1-1-hi', + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='hi', + translations={'content_%d' % i: {} for i in range(5)}, + ) + translation_1.update_timestamps() + translation_1.put() + + translation_2 = translation_models.EntityTranslationsModel( + id='exploration-exp_1-1-es', + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='es', + translations={'content_%d' % i: {} for i in range(10)}, + ) + translation_2.update_timestamps() + translation_2.put() + + self.assert_job_output_is( + [ + job_run_result.JobRunResult.as_stdout( + 'SUCCESS - Exploration %s counts are valid.' % exp_id + ) + ] + ) + + def test_mismatch(self) -> None: + exp_id = 'exp_2' + summary_model_2 = opportunity_models.ExplorationOpportunitySummaryModel( + id=exp_id, + topic_id='topic1', + topic_name='Topic 1', + story_id='story_1', + story_title='Story 1', + chapter_title='Chapter 1', + content_count=10, + incomplete_translation_language_codes=['hi'], + translation_counts={'hi': 6}, + language_codes_needing_voice_artists=[], + language_codes_with_assigned_voice_artists=[], + ) + summary_model_2.update_timestamps() + summary_model_2.put() + + translation_1_mock = translation_models.EntityTranslationsModel( + id='exploration-exp_2-1-hi', + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='hi', + translations={'content_%d' % i: {} for i in range(4)}, + ) + translation_1_mock.update_timestamps() + translation_1_mock.put() + + self.assert_job_output_is( + [ + job_run_result.JobRunResult.as_stderr( + 'Mismatch for exploration %s in hi: ' + 'stored=6, actual=4' % exp_id + ) + ] + ) + + def test_no_opportunity_summary_for_translation(self) -> None: + """Test that translations without a corresponding opportunity summary + produce no output. + """ + exp_id = 'exp_no_summary' + translation_model = translation_models.EntityTranslationsModel( + id='exploration-%s-1-hi' % exp_id, + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='hi', + translations={'content_0': {}, 'content_1': {}}, + ) + translation_model.update_timestamps() + translation_model.put() + + self.assert_job_output_is_empty() + + def test_mismatch_language_in_translations_but_not_in_summary( + self, + ) -> None: + """Test that a language present in EntityTranslationsModel but missing + from the opportunity summary's translation_counts is reported as a + mismatch. + """ + exp_id = 'exp_extra_lang' + summary_model = opportunity_models.ExplorationOpportunitySummaryModel( + id=exp_id, + topic_id='topic1', + topic_name='Topic 1', + story_id='story_1', + story_title='Story 1', + chapter_title='Chapter 1', + content_count=10, + incomplete_translation_language_codes=['hi'], + translation_counts={'hi': 3}, + language_codes_needing_voice_artists=[], + language_codes_with_assigned_voice_artists=[], + ) + summary_model.update_timestamps() + summary_model.put() + + translation_hi = translation_models.EntityTranslationsModel( + id='exploration-%s-1-hi' % exp_id, + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='hi', + translations={'content_%d' % i: {} for i in range(3)}, + ) + translation_hi.update_timestamps() + translation_hi.put() + + # 'es' translations exist but are not in the summary's + # translation_counts. + translation_es = translation_models.EntityTranslationsModel( + id='exploration-%s-1-es' % exp_id, + entity_id=exp_id, + entity_type='exploration', + entity_version=1, + language_code='es', + translations={'content_0': {}, 'content_1': {}}, + ) + translation_es.update_timestamps() + translation_es.put() + + self.assert_job_output_is( + [ + job_run_result.JobRunResult.as_stderr( + 'Mismatch for exploration %s in es: ' + 'stored=0 (missing), actual=2' % exp_id + ) + ] + ) diff --git a/core/jobs/batch_jobs/voiceover_synthesis_jobs.py b/core/jobs/batch_jobs/voiceover_synthesis_jobs.py new file mode 100644 index 0000000000000..0ddf4126a7041 --- /dev/null +++ b/core/jobs/batch_jobs/voiceover_synthesis_jobs.py @@ -0,0 +1,597 @@ +# coding: utf-8 +# +# Copyright 2025 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Jobs used for regenerating voiceovers for all the curated explorations.""" + +from __future__ import annotations + +import collections +import logging +import traceback + +from core import feconf +from core.domain import ( + exp_fetchers, + opportunity_services, + translation_fetchers, + voiceover_domain, + voiceover_regeneration_services, + voiceover_services, +) +from core.jobs import base_jobs, job_options +from core.jobs.io import ndb_io +from core.jobs.types import job_run_result +from core.platform import models + +import apache_beam as beam +from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Union, cast + +MYPY = False +if MYPY: # pragma: no cover + from mypy_imports import ( + datastore_services, + exp_models, + translation_models, + voiceover_models, + ) + +(exp_models, translation_models, voiceover_models) = ( + models.Registry.import_models( + [ + models.Names.EXPLORATION, + models.Names.TRANSLATION, + models.Names.VOICEOVER, + ] + ) +) +datastore_services = models.Registry.import_datastore_services() + + +# TODO(#15613): Here we use MyPy ignore because the incomplete typing of +# apache_beam library and absences of stubs in Typeshed, forces MyPy to +# assume that PTransform class is of type Any. Thus to avoid MyPy's error +# (Class cannot subclass 'PTransform' (has type 'Any')), we added an +# ignore here. +class GenerateVoiceoversFn(beam.DoFn): # type: ignore[misc] + """A DoFn that generates voiceovers for a given exploration.""" + + def __init__(self, oppia_project_id: Optional[str] = None) -> None: + super().__init__() + logging.info( + 'Voiceover synthesis log: Initializing GenerateVoiceoversFn.' + ) + + self.oppia_project_id = oppia_project_id + logging.info( + 'Voiceover synthesis log: Setting oppia project ID from args: %s', + self.oppia_project_id, + ) + + def process( + self, + combined_models: Tuple[ + str, + Dict[ + str, + Sequence[exp_models.ExplorationModel] + | Sequence[voiceover_models.EntityVoiceoversModel] + | Sequence[translation_models.EntityTranslationsModel], + ], + ], + autogeneration_policy_model: ( + voiceover_models.VoiceoverAutogenerationPolicyModel + ), + ) -> Iterator[ + Union[ + voiceover_models.EntityVoiceoversModel, + beam.pvalue.TaggedOutput[str], + ] + ]: + """Method to process each element in the PCollection. + + Args: + combined_models: tuple(str, dict). A tuple where the first element + is the exploration ID and the second element is a dictionary + with keys 'exploration', 'translations' and 'voiceovers' mapping + to a list of corresponding models. + autogeneration_policy_model: VoiceoverAutogenerationPolicyModel. + The voiceover autogeneration policy model. + + Yields: + EntityVoiceoversModel. The generated entity voiceover models. + str. The status string for the voiceover generation process. + """ + entity_id = combined_models[0] + + logging.info( + 'Voiceover synthesis log: Generating voiceovers for exploration ID: %s', + entity_id, + ) + # Here we use cast because we are narrowing down the type of + # exploration field in combined_models to Exploration model. + exploration_model = cast( + exp_models.ExplorationModel, combined_models[1]['exploration'][0] + ) + # Here we use cast because we are narrowing down the type of + # translations field in combined_models to Sequence of + # EntityTranslationsModel. + entity_translation_models = cast( + Sequence[translation_models.EntityTranslationsModel], + combined_models[1]['translations'], + ) + # Here we use cast because we are narrowing down the type of + # voiceovers field in combined_models to Sequence of EntityVoiceoversModel. + entity_voiceover_models = cast( + Sequence[voiceover_models.EntityVoiceoversModel], + combined_models[1]['voiceovers'], + ) + + entity_voiceovers_list, status_string = ( + VoiceoverSynthesisJob.generate_voiceovers_for_exploration( + exploration_model=exploration_model, + entity_translation_models=entity_translation_models, + entity_voiceover_models=entity_voiceover_models, + voiceover_policy_model=autogeneration_policy_model, + oppia_project_id=self.oppia_project_id, + ) + ) + + logging.info( + 'Voiceover synthesis log: Completed generating voiceovers for exploration ID: %s', + entity_id, + ) + + # Yield entity voiceovers to main output. + for entity_voiceovers in entity_voiceovers_list: + yield entity_voiceovers + + # Yield status string to tagged side output. + yield beam.pvalue.TaggedOutput('status', status_string) + + +class VoiceoverSynthesisJob(base_jobs.JobBase): + """A one-off job to generate voiceovers for all curated explorations in + English and other supported translated languages. + """ + + DATASTORE_UPDATES_ALLOWED = True + + @staticmethod + def is_exploration_curated(exploration_id: str) -> Optional[bool]: + """Checks whether the provided exploration ID belongs to a curated + exploration or not. + + Args: + exploration_id: str. The given exploration ID. + + Returns: + bool. A boolean value indicating if the exploration is curated + or not. + """ + try: + with datastore_services.get_ndb_context(): + return opportunity_services.is_exploration_available_for_contribution( + exploration_id + ) + except Exception: + logging.exception( + 'Not able to check whether exploration is curated or not' + ' for exploration ID %s.' % exploration_id + ) + return False + + @classmethod + def generate_voiceovers_for_exploration( + cls, + exploration_model: exp_models.ExplorationModel, + entity_translation_models: Sequence[ + translation_models.EntityTranslationsModel + ], + entity_voiceover_models: Sequence[ + voiceover_models.EntityVoiceoversModel + ], + voiceover_policy_model: voiceover_models.VoiceoverAutogenerationPolicyModel, + oppia_project_id: Optional[str] = None, + ) -> Tuple[Sequence[voiceover_models.EntityVoiceoversModel], str]: + """Generates voiceovers in English and all translated languages, + covering every supported accent for the given exploration. + + Args: + exploration_model: ExplorationModel. The exploration model for which + to generate voiceovers. + entity_translation_models: list(EntityTranslationsModel). The + existing entity translation models related to the exploration. + entity_voiceover_models: list(EntityVoiceoversModel). The existing + entity voiceover models related to the exploration. + voiceover_policy_model: VoiceoverAutogenerationPolicyModel. The + voiceover autogeneration policy model. + oppia_project_id: Optional[str]. The Google Cloud Project ID. + Explicitly required when running on Beam Dataflow, as workers + cannot retrieve the ID from environment variables. + + Returns: + Iterable[EntityVoiceoversModel]. An iterable of + EntityVoiceoversModels that were updated or created. + """ + logs_during_voiceover_generation = '' + + entity_translations_list = [] + entity_voiceovers_list = [] + + with datastore_services.get_ndb_context(): + # Converting exploration model to domain object. + exploration = exp_fetchers.get_exploration_from_model( + exploration_model, False + ) + logging.info( + 'Voiceover synthesis log: Converted exploration model to exploration domain object.' + ) + + # Converting EntityTranslationsModels to domain objects. + for entity_translation_model in list(entity_translation_models): + entity_translations_list.append( + translation_fetchers.get_entity_translation_from_model( + entity_translation_model + ) + ) + logging.info( + 'Voiceover synthesis log: Converted entity translation models to ' + 'entity translation domain objects.' + ) + + # Converting EntityVoiceoversModels to domain objects. + for entity_voiceover_model in list(entity_voiceover_models): + if entity_voiceover_model.entity_version != exploration.version: + continue + entity_voiceovers_list.append( + voiceover_services.get_entity_voiceovers_from_model( + entity_voiceover_model + ) + ) + logging.info( + 'Voiceover synthesis log: Converted entity voiceover models to ' + 'entity voiceover domain objects.' + ) + + # Extracting language codes mapping from the autogeneration policy + # model. + language_codes_mapping = ( + voiceover_policy_model.language_codes_mapping + ) + + entity_type = feconf.ENTITY_TYPE_EXPLORATION + entity_id = exploration.id + entity_version = exploration.version + + # A dictionary that maps each entity voiceover ID to its corresponding + # EntityVoiceovers domain object. + entity_voiceovers_id_to_domain_object = {} + + for entity_voiceovers in entity_voiceovers_list: + entity_voiceovers_id = '%s-%s-%s-%s' % ( + entity_voiceovers.entity_type, + entity_voiceovers.entity_id, + entity_voiceovers.entity_version, + entity_voiceovers.language_accent_code, + ) + + entity_voiceovers_id_to_domain_object[entity_voiceovers_id] = ( + entity_voiceovers + ) + + # A dictionary mapping each language code to a list of accent codes + # that support autogenerated voiceovers. + autogeneratable_language_codes_mapping: Dict[str, List[str]] = {} + + for language_code, accent_mapping in language_codes_mapping.items(): + autogeneratable_language_codes_mapping[language_code] = [] + for accent_code, is_autogeneratable in accent_mapping.items(): + if is_autogeneratable: + autogeneratable_language_codes_mapping[ + language_code + ].append(accent_code) + + # A dictionary where each key is a language code, and each value is a + # content mapping dictionary. The content mapping dictionary contains + # content IDs as keys and their corresponding HTML content as values. + language_code_to_contents_mapping = {} + + language_code_to_contents_mapping.update( + voiceover_services.extract_english_voiceover_texts_from_exploration( + exploration + ) + ) + language_code_to_contents_mapping.update( + voiceover_services.extract_translated_voiceover_texts_from_entity_translations( + entity_translations_list + ) + ) + + # Get all language codes that need voiceover regeneration in this + # request. + language_codes = list(language_code_to_contents_mapping.keys()) + + for language_code in language_codes: + language_accent_codes = autogeneratable_language_codes_mapping.get( + language_code, [] + ) + + content_ids_to_content_values = ( + language_code_to_contents_mapping.get(language_code, {}) + ) + + for language_accent_code in language_accent_codes: + entity_voiceovers_id = '%s-%s-%s-%s' % ( + entity_type, + entity_id, + str(entity_version), + language_accent_code, + ) + + default_entity_voiceovers = ( + voiceover_domain.EntityVoiceovers.create_empty( + entity_id, + entity_type, + entity_version, + language_accent_code, + ) + ) + error_message_to_content_ids_dict = collections.defaultdict( + list + ) + + entity_voiceovers = entity_voiceovers_id_to_domain_object.get( + entity_voiceovers_id, default_entity_voiceovers + ) + + logging.info( + 'Voiceover synthesis log: Generating voiceovers for Entityvoiceover with ID: %s.', + entity_voiceovers_id, + ) + logs_during_voiceover_generation += ( + 'EntityVoiceovers ID: %s.\n' % entity_voiceovers_id + ) + + number_of_content_ids = len(content_ids_to_content_values) + number_of_characters = 0 + + logging.info( + 'Voiceover synthesis log: content_ids_to_content_values: %s.', + content_ids_to_content_values, + ) + logging.info( + 'Voiceover synthesis log: number_of_content_ids: %s.', + number_of_content_ids, + ) + + for ( + content_id, + content_html, + ) in content_ids_to_content_values.items(): + + try: + voiceover_filename = voiceover_regeneration_services.generate_new_voiceover_filename( + content_id, language_accent_code + ) + logging.info( + 'Voiceover synthesis log: Generated new voiceover filename: %s for content_id: %s, content_html: %s.' + % (voiceover_filename, content_id, content_html) + ) + + with datastore_services.get_ndb_context(): + sentence_tokens_with_durations = voiceover_regeneration_services.synthesize_voiceover_for_html_string( + entity_id, + content_html, + language_accent_code, + voiceover_filename, + oppia_project_id, + ) + + if not sentence_tokens_with_durations: + continue + + voiceover = voiceover_regeneration_services.fetch_voiceover_by_filename( + entity_id, voiceover_filename, oppia_project_id + ) + + number_of_characters += len(content_html) + + entity_voiceovers.add_voiceover( + content_id, feconf.VoiceoverType.AUTO, voiceover + ) + entity_voiceovers.add_automated_voiceovers_audio_offsets( + content_id, sentence_tokens_with_durations + ) + + logging.info( + 'Voiceover synthesis log: Generated voiceover for content_id: %s.', + content_id, + ) + except Exception as error: + error_message_to_content_ids_dict[str(error)].append( + content_id + ) + stack_trace = traceback.format_exc() + logging.error( + 'Voiceover synthesis log: Stack trace: %s', + stack_trace, + ) + logging.error( + 'Voiceover synthesis log: Error generating voiceover for exploration ID: %s, language_accent_code: %s, content_id: %s. Error: %s' + % ( + entity_id, + language_accent_code, + content_id, + str(error), + ) + ) + + for ( + error_message, + content_ids, + ) in error_message_to_content_ids_dict.items(): + comma_separated_content_ids = ', '.join(content_ids) + logs_during_voiceover_generation += ( + 'Content IDs failed: [%s]. Error message: %s\n' + % (comma_separated_content_ids, error_message) + ) + + entity_voiceovers.validate() + entity_voiceovers_id_to_domain_object[entity_voiceovers_id] = ( + entity_voiceovers + ) + + final_report_logs = ( + 'Total content IDs processed: %d. ' + 'Total characters processed: %d.\n' + % ( + number_of_content_ids, + number_of_characters, + ) + ) + logging.info('Voiceover synthesis log: %s.' % final_report_logs) + logs_during_voiceover_generation += final_report_logs + logging.info( + 'Voiceover synthesis log: Completed voiceover generation for entity ID: %s.' + % entity_voiceovers_id + ) + + # List of EntityVoiceoversModel instances to be stored in the datastore. + entity_voiceover_models_to_put = [] + for entity_voiceovers in entity_voiceovers_id_to_domain_object.values(): + with datastore_services.get_ndb_context(): + entity_voiceover_models_to_put.append( + voiceover_services.create_entity_voiceovers_model( + entity_voiceovers + ) + ) + + return ( + entity_voiceover_models_to_put, + logs_during_voiceover_generation, + ) + + def run(self) -> beam.PCollection[job_run_result.JobRunResult]: + """Returns a PCollection of job run results for EntityVoiceoversModels + that were updated after voiceover synthesis. + + Returns: + beam.PCollection[job_run_result.JobRunResult]. A PCollection + containing job run results with the IDs of the + EntityVoiceoversModels that were updated or created. + """ + exploration_models = ( + self.pipeline + | 'Get exploration models' + >> ndb_io.GetModels(exp_models.ExplorationModel.get_all()) + | 'Filter out curated explorations' + >> beam.Filter( + lambda model: self.is_exploration_curated( + exploration_id=model.id + ) + ) + ) + + entity_translation_models = ( + self.pipeline + | 'Get all entity translation models' + >> ndb_io.GetModels( + translation_models.EntityTranslationsModel.get_all() + ) + | 'Filter out entity translations for curated explorations' + >> beam.Filter( + lambda model: self.is_exploration_curated( + exploration_id=model.entity_id + ) + ) + ) + + entity_voiceovers_models = ( + self.pipeline + | 'Get all entity voiceover models' + >> ndb_io.GetModels( + voiceover_models.EntityVoiceoversModel.get_all() + ) + | 'Filter out entity voiceovers for curated explorations' + >> beam.Filter( + lambda model: self.is_exploration_curated( + exploration_id=model.entity_id + ) + ) + ) + + exploration_id_to_exploration = ( + exploration_models + | 'Map exploration ID to exploration model' + >> beam.Map(lambda model: (model.id, model)) + ) + + entity_id_to_translation_models = ( + entity_translation_models + | 'Map entity ID to translation model' + >> beam.Map(lambda model: (model.entity_id, model)) + ) + + entity_id_to_voiceover_models = ( + entity_voiceovers_models + | 'Map entity ID to voiceover model' + >> beam.Map(lambda model: (model.entity_id, model)) + ) + + combined_models = { + 'exploration': exploration_id_to_exploration, + 'translations': entity_id_to_translation_models, + 'voiceovers': entity_id_to_voiceover_models, + } | 'Join all by entity ID' >> beam.CoGroupByKey() + + voiceover_policy_model = ( + self.pipeline + | 'Get all voiceover autogeneration policy models' + >> ndb_io.GetModels( + voiceover_models.VoiceoverAutogenerationPolicyModel.get_all() + ) + ) + + custom_options = self.pipeline.options.view_as(job_options.JobOptions) + oppia_project_id = custom_options.oppia_project_id + + voiceovers_and_status = ( + combined_models + | 'Generate voiceovers for each exploration' + >> beam.ParDo( + GenerateVoiceoversFn(oppia_project_id=oppia_project_id), + beam.pvalue.AsSingleton(voiceover_policy_model), + ).with_outputs('status', main='voiceovers') + ) + + entity_voiceovers_models = voiceovers_and_status.voiceovers + status_strings = voiceovers_and_status.status + + if self.DATASTORE_UPDATES_ALLOWED: + unused_put_results = ( + entity_voiceovers_models + | 'Put models into datastore' >> ndb_io.PutModels() + ) + + return status_strings | 'Format results' >> beam.Map( + job_run_result.JobRunResult.as_stdout + ) + + +class VoiceoverSynthesisAuditJob(VoiceoverSynthesisJob): + """Audit job for VoiceoverSynthesisJob.""" + + DATASTORE_UPDATES_ALLOWED = False diff --git a/core/jobs/batch_jobs/voiceover_synthesis_jobs_test.py b/core/jobs/batch_jobs/voiceover_synthesis_jobs_test.py new file mode 100644 index 0000000000000..a69fb04fa7387 --- /dev/null +++ b/core/jobs/batch_jobs/voiceover_synthesis_jobs_test.py @@ -0,0 +1,604 @@ +# coding: utf-8 +# +# Copyright 2025 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for jobs.batch_jobs.voiceover_synthesis_jobs.""" + +from __future__ import annotations + +from core import constants, feconf +from core.domain import ( + exp_domain, + exp_services, + state_domain, + story_domain, + story_services, + topic_domain, + topic_services, + voiceover_regeneration_services, +) +from core.jobs import job_test_utils +from core.jobs.batch_jobs import voiceover_synthesis_jobs +from core.jobs.types import job_run_result +from core.platform import models +from core.tests import test_utils + +from typing import Dict, List, Type, Union + +MYPY = False +if MYPY: + from mypy_imports import translation_models, voiceover_models + +(translation_models, voiceover_models) = models.Registry.import_models( + [models.Names.TRANSLATION, models.Names.VOICEOVER] +) + + +class VoiceoverSynthesisBaseClass( + job_test_utils.JobTestBase, test_utils.GenericTestBase +): + """Base class for voiceover synthesis job tests.""" + + EDITOR_EMAIL_1 = 'editor1@example.com' + EDITOR_EMAIL_2 = 'editor2@example.com' + EDITOR_USERNAME_1 = 'editor1' + EDITOR_USERNAME_2 = 'editor2' + + CURATED_EXPLORATION_ID_1 = 'exploration_id_1' + CURATED_EXPLORATION_ID_2 = 'exploration_id_2' + NON_CURATED_EXPLORATION_ID = 'exploration_id_3' + + TOPIC_ID_1 = 'topic_id_1' + TOPIC_ID_2 = 'topic_id_2' + STORY_ID_1 = 'story_id_1' + STORY_ID_2 = 'story_id_2' + + def setUp(self) -> None: + super().setUp() + self.signup(self.EDITOR_EMAIL_1, self.EDITOR_USERNAME_1) + self.signup(self.EDITOR_EMAIL_2, self.EDITOR_USERNAME_2) + self.signup(self.CURRICULUM_ADMIN_EMAIL, self.CURRICULUM_ADMIN_USERNAME) + self.signup(self.OWNER_EMAIL, self.OWNER_USERNAME) + + self.set_curriculum_admins( + [ + self.EDITOR_USERNAME_1, + self.EDITOR_USERNAME_2, + self.CURRICULUM_ADMIN_USERNAME, + ] + ) + + self.admin_id = self.get_user_id_from_email(self.CURRICULUM_ADMIN_EMAIL) + self.owner_id = self.get_user_id_from_email(self.OWNER_EMAIL) + + self.voiceover_dict_1: state_domain.VoiceoverDict = { + 'filename': 'filename1.mp3', + 'file_size_bytes': 3000, + 'needs_update': False, + 'duration_secs': 42.43, + } + self.voiceover_dict_2: state_domain.VoiceoverDict = { + 'filename': 'filename2.mp3', + 'file_size_bytes': 3000, + 'needs_update': False, + 'duration_secs': 40, + } + + def _create_data_for_testing(self) -> None: + """This method creates three explorations — two curated and one + non-curated. It adds Hindi translations to the first curated + exploration and Portuguese translations to the second. Additionally, + it adds voiceovers in Hindi, Portuguese, and English to selected + content within the curated explorations. + """ + + # Creating and publishing two topics. + topic_1 = topic_domain.Topic.create_default_topic( + self.TOPIC_ID_1, 'topic1', 'abbrev', 'description', 'fragm' + ) + topic_1.thumbnail_filename = 'thumbnail.svg' + topic_1.thumbnail_bg_color = '#C6DCDA' + topic_1.subtopics = [ + topic_domain.Subtopic( + 1, + 'Title', + ['skill_id_1'], + 'image.svg', + constants.constants.ALLOWED_THUMBNAIL_BG_COLORS['subtopic'][0], + 21131, + 'dummy-subtopic-url', + ) + ] + topic_1.next_subtopic_id = 2 + topic_1.skill_ids_for_diagnostic_test = ['skill_id_1'] + + topic_services.save_new_topic(self.owner_id, topic_1) + topic_services.publish_topic(self.TOPIC_ID_1, self.admin_id) + + story_1 = story_domain.Story.create_default_story( + self.STORY_ID_1, + 'A story', + 'Description', + self.TOPIC_ID_1, + 'story-two', + ) + story_services.save_new_story(self.owner_id, story_1) + topic_services.add_canonical_story( + self.owner_id, self.TOPIC_ID_1, self.STORY_ID_1 + ) + + topic_services.publish_story( + self.TOPIC_ID_1, self.STORY_ID_1, self.admin_id + ) + + topic_2 = topic_domain.Topic.create_default_topic( + self.TOPIC_ID_2, 'topic2', 'abbrev-top', 'description', 'fragmem' + ) + topic_2.thumbnail_filename = 'thumbnail.svg' + topic_2.thumbnail_bg_color = '#C6DCDA' + topic_2.subtopics = [ + topic_domain.Subtopic( + 1, + 'Title subtopic', + ['skill_id_1'], + 'image.svg', + constants.constants.ALLOWED_THUMBNAIL_BG_COLORS['subtopic'][0], + 21131, + 'dummy-subtopic-url-sub', + ) + ] + topic_2.next_subtopic_id = 2 + topic_2.skill_ids_for_diagnostic_test = ['skill_id_1'] + + topic_services.save_new_topic(self.owner_id, topic_2) + topic_services.publish_topic(self.TOPIC_ID_2, self.admin_id) + + story_2 = story_domain.Story.create_default_story( + self.STORY_ID_2, + 'The second story', + 'Description second', + self.TOPIC_ID_2, + 'story-three', + ) + story_services.save_new_story(self.owner_id, story_2) + topic_services.add_canonical_story( + self.owner_id, self.TOPIC_ID_2, self.STORY_ID_2 + ) + + topic_services.publish_story( + self.TOPIC_ID_2, self.STORY_ID_2, self.admin_id + ) + + # Creating 2 curated explorations. + exploration_1 = self.save_new_valid_exploration( + self.CURATED_EXPLORATION_ID_1, + self.owner_id, + title='title1', + category=constants.constants.ALL_CATEGORIES[0], + end_state_name='End State', + ) + + self.publish_exploration(self.owner_id, exploration_1.id) + + exp_services.update_exploration( + self.owner_id, + self.CURATED_EXPLORATION_ID_1, + [ + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'Introduction', + 'new_value': { + 'content_id': 'content_0', + 'html': 'This is the first card of first exploration.
', + }, + } + ), + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'End State', + 'new_value': { + 'content_id': 'content_3', + 'html': 'This is the last card of first exploration.
', + }, + } + ), + ], + 'Changes content.', + ) + + story_services.update_story( + self.owner_id, + self.STORY_ID_1, + [ + story_domain.StoryChange( + { + 'cmd': 'add_story_node', + 'node_id': 'node_1', + 'title': 'Node1', + } + ), + story_domain.StoryChange( + { + 'cmd': 'update_story_node_property', + 'property_name': 'exploration_id', + 'node_id': 'node_1', + 'old_value': None, + 'new_value': self.CURATED_EXPLORATION_ID_1, + } + ), + ], + 'Changes.', + ) + + exploration_2 = self.save_new_valid_exploration( + self.CURATED_EXPLORATION_ID_2, + self.owner_id, + title='title2', + category=constants.constants.ALL_CATEGORIES[0], + end_state_name='End State', + ) + self.publish_exploration(self.owner_id, exploration_2.id) + + exp_services.update_exploration( + self.owner_id, + self.CURATED_EXPLORATION_ID_2, + [ + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'Introduction', + 'new_value': { + 'content_id': 'content_0', + 'html': 'This is the first card of second exploration.
', + }, + } + ), + exp_domain.ExplorationChange( + { + 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, + 'property_name': exp_domain.STATE_PROPERTY_CONTENT, + 'state_name': 'End State', + 'new_value': { + 'content_id': 'content_3', + 'html': 'This is the last card of second exploration.
', + }, + } + ), + ], + 'Changes content.', + ) + + story_services.update_story( + self.owner_id, + self.STORY_ID_2, + [ + story_domain.StoryChange( + { + 'cmd': 'add_story_node', + 'node_id': 'node_1', + 'title': 'Node1', + } + ), + story_domain.StoryChange( + { + 'cmd': 'update_story_node_property', + 'property_name': 'exploration_id', + 'node_id': 'node_1', + 'old_value': None, + 'new_value': self.CURATED_EXPLORATION_ID_2, + } + ), + ], + 'Changes.', + ) + + # Create a non-curated exploration. + exploration_3 = self.save_new_valid_exploration( + self.NON_CURATED_EXPLORATION_ID, + self.owner_id, + title='title3', + category=constants.constants.ALL_CATEGORIES[0], + end_state_name='End State', + ) + self.publish_exploration(self.owner_id, exploration_3.id) + + # Adding Hindi and Portuguese translations to the first and second. + translation_models.EntityTranslationsModel.create_new( + 'exploration', + self.CURATED_EXPLORATION_ID_1, + 2, + 'hi', + { + 'content_0': { + 'content_value': 'यह प्रथम अन्वेषण का पहला कार्ड है.
', + 'content_format': 'html', + 'needs_update': False, + }, + 'content_3': { + 'content_value': 'यह प्रथम अन्वेषण का अंतिम कार्ड है.
', + 'content_format': 'html', + 'needs_update': True, + }, + }, + ).put() + + translation_models.EntityTranslationsModel.create_new( + 'exploration', + self.CURATED_EXPLORATION_ID_2, + 2, + 'pt', + { + 'content_0': { + 'content_value': 'Esta é a primeira carta da segunda exploração.
', + 'content_format': 'html', + 'needs_update': False, + }, + 'content_3': { + 'content_value': 'Esta é a segunda carta da segunda exploração.
', + 'content_format': 'html', + 'needs_update': True, + }, + }, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_1, + 1, + 'en-US', + { + 'content_0': { + 'manual': None, + 'auto': None, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_1, + 2, + 'en-US', + { + 'content_0': { + 'manual': self.voiceover_dict_1, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_1, + 2, + 'hi-IN', + { + 'content_0': { + 'manual': None, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_2, + 2, + 'en-US', + { + 'content_0': { + 'manual': self.voiceover_dict_1, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_models.EntityVoiceoversModel.create_new( + feconf.ENTITY_TYPE_EXPLORATION, + self.CURATED_EXPLORATION_ID_2, + 2, + 'pt-BR', + { + 'content_0': { + 'manual': None, + 'auto': self.voiceover_dict_2, + } + }, + {}, + ).put() + + voiceover_autogeneration_policy_model = ( + voiceover_models.VoiceoverAutogenerationPolicyModel( + id=voiceover_models.VOICEOVER_AUTOGENERATION_POLICY_ID + ) + ) + voiceover_autogeneration_policy_model.language_codes_mapping = { + 'en': {'en-US': True, 'en-NG': False}, + 'hi': {'hi-IN': True}, + 'pt': {'pt-BR': True}, + } + ( + voiceover_autogeneration_policy_model.autogenerated_voiceovers_are_enabled + ) = True + voiceover_autogeneration_policy_model.update_timestamps() + voiceover_autogeneration_policy_model.put() + + +class VoiceoverSynthesisJobRunTests(VoiceoverSynthesisBaseClass): + + JOB_CLASS: Type[voiceover_synthesis_jobs.VoiceoverSynthesisJob] = ( + voiceover_synthesis_jobs.VoiceoverSynthesisJob + ) + + def test_empty_storage(self) -> None: + self.assert_job_output_is_empty() + + def test_should_regenerate_voiceover_successfully(self) -> None: + self._create_data_for_testing() + + expected_output_1 = ( + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 101.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-hi-IN.\n' + 'Total content IDs processed: 1. Total characters processed: 41.\n' + ) + + expected_output_2 = ( + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 103.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-pt-BR.\n' + 'Total content IDs processed: 1. Total characters processed: 53.\n' + ) + + expected_output = [ + job_run_result.JobRunResult(stdout=expected_output_1, stderr=''), + job_run_result.JobRunResult(stdout=expected_output_2, stderr=''), + ] + + self.assert_job_output_is(expected_output) + + def test_check_is_exploration_curated_for_invalid_id(self) -> None: + is_exploration_curated = voiceover_synthesis_jobs.VoiceoverSynthesisJob.is_exploration_curated( + exploration_id='' + ) + self.assertFalse(is_exploration_curated) + + def test_should_handle_failures_during_voiceover_regneration(self) -> None: + def mock_synthesize_voiceover_for_html_string( + _exploration_id: str, + _content_html: str, + _language_accent_code: str, + _voiceover_filename: str, + _oppia_project_id: str, + ) -> List[Dict[str, Union[str, float]]]: + raise Exception('Failed to generate voiceovers.') + + self._create_data_for_testing() + + expected_output_1 = ( + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Content IDs failed: [content_0, default_outcome_1, ca_placeholder_2, content_3]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-hi-IN.\n' + 'Content IDs failed: [content_0]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 1. Total characters processed: 0.\n' + ) + expected_output_2 = ( + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Content IDs failed: [content_0, default_outcome_1, ca_placeholder_2, content_3]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-pt-BR.\n' + 'Content IDs failed: [content_0]. Error message: Failed to generate voiceovers.\n' + 'Total content IDs processed: 1. Total characters processed: 0.\n' + ) + + with self.swap( + voiceover_regeneration_services, + 'synthesize_voiceover_for_html_string', + mock_synthesize_voiceover_for_html_string, + ): + expected_output = [ + job_run_result.JobRunResult( + stdout=expected_output_1, stderr='' + ), + job_run_result.JobRunResult( + stdout=expected_output_2, stderr='' + ), + ] + + self.assert_job_output_is(expected_output) + + def test_should_handle_empty_strings_during_voiceover_regneration( + self, + ) -> None: + + def mock_synthesize_voiceover_for_html_string( + _exploration_id: str, + _content_html: str, + _language_accent_code: str, + _voiceover_filename: str, + _oppia_project_id: str, + ) -> List[Dict[str, Union[str, float]]]: + return [] + + self._create_data_for_testing() + + expected_output_1 = ( + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-hi-IN.\n' + 'Total content IDs processed: 1. Total characters processed: 0.\n' + ) + expected_output_2 = ( + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 0.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-pt-BR.\n' + 'Total content IDs processed: 1. Total characters processed: 0.\n' + ) + + with self.swap( + voiceover_regeneration_services, + 'synthesize_voiceover_for_html_string', + mock_synthesize_voiceover_for_html_string, + ): + expected_output = [ + job_run_result.JobRunResult( + stdout=expected_output_1, stderr='' + ), + job_run_result.JobRunResult( + stdout=expected_output_2, stderr='' + ), + ] + + self.assert_job_output_is(expected_output) + + +class VoiceoverSynthesisAuditJobRunTests(VoiceoverSynthesisBaseClass): + + JOB_CLASS: Type[voiceover_synthesis_jobs.VoiceoverSynthesisAuditJob] = ( + voiceover_synthesis_jobs.VoiceoverSynthesisAuditJob + ) + + def test_should_regenerate_voiceover_successfully(self) -> None: + self._create_data_for_testing() + + expected_output_1 = ( + 'EntityVoiceovers ID: exploration-exploration_id_1-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 101.\n' + 'EntityVoiceovers ID: exploration-exploration_id_1-2-hi-IN.\n' + 'Total content IDs processed: 1. Total characters processed: 41.\n' + ) + + expected_output_2 = ( + 'EntityVoiceovers ID: exploration-exploration_id_2-2-en-US.\n' + 'Total content IDs processed: 4. Total characters processed: 103.\n' + 'EntityVoiceovers ID: exploration-exploration_id_2-2-pt-BR.\n' + 'Total content IDs processed: 1. Total characters processed: 53.\n' + ) + + expected_output = [ + job_run_result.JobRunResult(stdout=expected_output_1, stderr=''), + job_run_result.JobRunResult(stdout=expected_output_2, stderr=''), + ] + self.assert_job_output_is(expected_output) diff --git a/core/jobs/job_options.py b/core/jobs/job_options.py index 6d9860de00ec2..113f13cdb72b3 100644 --- a/core/jobs/job_options.py +++ b/core/jobs/job_options.py @@ -46,10 +46,37 @@ class JobOptions(pipeline_options.PipelineOptions): # type: ignore[misc] str, 'Namespace for isolating the NDB operations during tests.', ), + 'oppia_project_id': ( + str, + 'The ID of the Google Cloud Project for Oppia.', + ), + } + + # A subset of Dataflow pipeline options related to resource utilization. + # For the complete list of available options, refer to: + # https://docs.cloud.google.com/dataflow/docs/reference/pipeline-options#resource_utilization + DATAFLOW_RESOURCE_OPTIONS = { + 'max_num_workers', + 'num_workers', + 'autoscaling_algorithm', + } + + # Historically, Beam jobs get data from the datastore and process it. + # In some cases, we may want to run parameterized Beam jobs that get data + # from other sources. For example, in the case of bulk regeneration of + # voiceovers by language accent, we want to pass language accent code as a + # parameter to the Beam job. + SPECIAL_JOB_OPTIONS = { + 'language_accent_code': ( + str, + 'Language-accent code to scope voiceover synthesis jobs.', + ), } def __init__( - self, flags: Optional[List[str]] = None, **job_options: Optional[str] + self, + flags: Optional[List[str]] = None, + **job_options: Optional[str | int], ) -> None: """Initializes a new JobOptions instance. @@ -64,13 +91,17 @@ def __init__( Raises: ValueError. Unsupported job option(s). """ - unsupported_options = set(job_options).difference(self.JOB_OPTIONS) + allowed_options = set(self.JOB_OPTIONS.keys()).union( + self.DATAFLOW_RESOURCE_OPTIONS, self.SPECIAL_JOB_OPTIONS + ) + unsupported_options = set(job_options).difference(allowed_options) if unsupported_options: joined_unsupported_options = ', '.join(sorted(unsupported_options)) raise ValueError( 'Unsupported option(s): %s' % joined_unsupported_options ) oppia_project_id = app_identity_services.get_application_id() + assert isinstance(oppia_project_id, str) super().__init__( # Needed by PipelineOptions. @@ -106,3 +137,11 @@ def _add_argparse_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument( '--%s' % option_name, help=option_doc, type=option_type ) + + for option_name, ( + option_type, + option_doc, + ) in cls.SPECIAL_JOB_OPTIONS.items(): + parser.add_argument( + '--%s' % option_name, help=option_doc, type=option_type + ) diff --git a/core/jobs/job_options_test.py b/core/jobs/job_options_test.py index 3a4a92ced4e27..c650c04e09dd2 100644 --- a/core/jobs/job_options_test.py +++ b/core/jobs/job_options_test.py @@ -34,6 +34,11 @@ def test_overwritten_values(self) -> None: self.assertEqual(options.namespace, 'abc') + def test_special_overwritten_values(self) -> None: + options = job_options.JobOptions(language_accent_code='en-IN') + + self.assertEqual(options.language_accent_code, 'en-IN') + def test_unsupported_values(self) -> None: with self.assertRaisesRegex(ValueError, r'Unsupported option\(s\)'): job_options.JobOptions(a='a', b='b') diff --git a/core/jobs/job_test_utils.py b/core/jobs/job_test_utils.py index 9264aaef6b136..e91af3b973135 100644 --- a/core/jobs/job_test_utils.py +++ b/core/jobs/job_test_utils.py @@ -63,7 +63,10 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.pipeline = test_pipeline.TestPipeline( runner=runners.DirectRunner(), - options=job_options.JobOptions(namespace=self.namespace), + options=job_options.JobOptions( + namespace=self.namespace, + oppia_project_id='dev-project-id', + ), ) self._pipeline_context_stack: Optional[contextlib.ExitStack] = None diff --git a/core/jobs/jobs_manager.py b/core/jobs/jobs_manager.py index c719e741d7b25..d8f8ec89c2250 100644 --- a/core/jobs/jobs_manager.py +++ b/core/jobs/jobs_manager.py @@ -33,7 +33,7 @@ import apache_beam as beam from apache_beam import runners from google.cloud import dataflow -from typing import Iterator, Optional, Type +from typing import Dict, Iterator, Optional, Type MYPY = False if MYPY: # pragma: no cover @@ -103,6 +103,7 @@ def run_job( sync: bool, namespace: Optional[str] = None, pipeline: Optional[beam.Pipeline] = None, + parameterized_args: Optional[Dict[str, str]] = None, ) -> beam_job_models.BeamJobRunModel: """Runs the specified job synchronously. @@ -115,6 +116,8 @@ def run_job( namespace: str. The namespace in which models should be created. pipeline: Pipeline. The pipeline to run the job upon. If omitted, then a new pipeline will be used instead. + parameterized_args: dict(str, str). The dictionary of parameterized + arguments to be passed to the job. Returns: BeamJobRun. Contains metadata related to the execution status of the @@ -123,14 +126,34 @@ def run_job( Raises: RuntimeError. Failed to deploy given job to the Dataflow service. """ + job_name = job_class.__name__ + + additional_options: Dict[str, int | str] = {} + if does_job_requires_limiting_workers(job_name): + # We want to limit the number of workers for Beam jobs related to voiceover + # synthesis, as these jobs depend on Azure for voiceover regeneration, and + # increasing parallelism may lead to rate-limiting issues. + logging.info('Limiting the number of workers for job: %s' % job_name) + additional_options = { + 'max_num_workers': 15, + 'autoscaling_algorithm': 'THROUGHPUT_BASED', + } + + if parameterized_args: + additional_options.update(parameterized_args) + if pipeline is None: pipeline = beam.Pipeline( runner=runners.DirectRunner() if sync else runners.DataflowRunner(), - options=job_options.JobOptions(namespace=namespace), + options=job_options.JobOptions( + flags=None, + namespace=namespace, + oppia_project_id=app_identity_services.get_application_id(), + **additional_options, + ), ) job = job_class(pipeline) - job_name = job_class.__name__ # Clear cache before running the job to be sure that the cache # does not affect the job. @@ -294,3 +317,22 @@ def _put_job_stderr(job_id: str, stderr: str) -> None: job_id, '', stderr ) result_model.put() + + +def does_job_requires_limiting_workers(job_name: str) -> bool: + """Returns whether the given job requires limiting the number of workers. + + Args: + job_name: str. The name of the job. + + Returns: + bool. Whether the given job requires limiting the number of workers. + """ + jobs_requiring_limiting_workers = [ + 'VoiceoverSynthesisJob', + 'VoiceoverSynthesisAuditJob', + 'VoiceoverSynthesisByAccentJob', + # The below job is used in unit tests. + 'VoiceoverSynthesisForTestingJob', + ] + return job_name in jobs_requiring_limiting_workers diff --git a/core/jobs/jobs_manager_test.py b/core/jobs/jobs_manager_test.py index 580e843201ff9..6882fd946116a 100644 --- a/core/jobs/jobs_manager_test.py +++ b/core/jobs/jobs_manager_test.py @@ -43,6 +43,15 @@ def run(self) -> beam.PCollection[job_run_result.JobRunResult]: ) +class VoiceoverSynthesisForTestingJob(base_jobs.JobBase): + """Simple job for voiceover synthesis to test resource limiting.""" + + def run(self) -> beam.PCollection[job_run_result.JobRunResult]: + return self.pipeline | beam.Create( + [job_run_result.JobRunResult(stdout='o', stderr='e')] + ) + + class FailingJob(base_jobs.JobBase): """Simple job that always raises an exception.""" @@ -113,6 +122,24 @@ def test_async_job_that_does_not_start(self) -> None: result = beam_job_services.get_beam_job_run_result(run.id) self.assertIn('Failed to deploy WorkingJob', result.stderr) + def test_job_run_with_parameterized_arg(self) -> None: + run = jobs_manager.run_job( + WorkingJob, + True, + namespace=self.namespace, + parameterized_args={'language_accent_code': 'en-IN'}, + ) + + self.assertEqual(run.latest_job_state, 'DONE') + + run_model = beam_job_models.BeamJobRunModel.get(run.id) + self.assertEqual(run, run_model) + + self.assertEqual( + beam_job_services.get_beam_job_run_result(run.id).to_dict(), + {'stdout': 'o', 'stderr': 'e'}, + ) + class RefreshStateOfBeamJobRunModelTests(test_utils.GenericTestBase): @@ -253,3 +280,33 @@ def test_failed_api_call_logs_the_exception(self) -> None: self.assertGreater(len(logs), 0) self.assertIn('uh-oh', logs[0]) + + +class LimitJobResourcesTests(test_utils.GenericTestBase): + + def test_does_job_requires_limiting_workers_true(self) -> None: + job_name = 'VoiceoverSynthesisJob' + self.assertTrue( + jobs_manager.does_job_requires_limiting_workers(job_name) + ) + + def test_does_job_requires_limiting_workers_false(self) -> None: + job_name = 'SomeOtherJob' + self.assertFalse( + jobs_manager.does_job_requires_limiting_workers(job_name) + ) + + def test_working_voiceover_sync_job(self) -> None: + run = jobs_manager.run_job( + VoiceoverSynthesisForTestingJob, True, namespace=self.namespace + ) + + self.assertEqual(run.latest_job_state, 'DONE') + + run_model = beam_job_models.BeamJobRunModel.get(run.id) + self.assertEqual(run, run_model) + + self.assertEqual( + beam_job_services.get_beam_job_run_result(run.id).to_dict(), + {'stdout': 'o', 'stderr': 'e'}, + ) diff --git a/core/jobs/registry.py b/core/jobs/registry.py index 3fa1798fb5548..42038ad60ce21 100644 --- a/core/jobs/registry.py +++ b/core/jobs/registry.py @@ -45,8 +45,11 @@ audit_non_existent_threads_messages_jobs, audit_stories_with_disconnected_node_ids_job, audit_threads_with_missing_suggestions_jobs, + blog_author_details_migration_jobs, blog_post_search_indexing_jobs, blog_validation_jobs, + cleanup_duplicate_translation_suggestions_jobs, + cloud_task_run_migration_jobs, collection_info_jobs, contributor_admin_stats_jobs, delete_duplicate_content_ids_jobs, @@ -66,11 +69,14 @@ subtopic_migration_jobs, suggestion_migration_jobs, suggestion_stats_computation_jobs, + synthesize_voiceover_by_language_accent_jobs, topic_migration_jobs, + translation_audit_jobs, translation_migration_jobs, user_bios_change_jobs, user_stats_computation_jobs, user_validation_jobs, + voiceover_synthesis_jobs, ) from typing import List, Type diff --git a/core/platform/app_identity/gae_app_identity_services.py b/core/platform/app_identity/gae_app_identity_services.py index 14f21671cf523..4b199035789df 100644 --- a/core/platform/app_identity/gae_app_identity_services.py +++ b/core/platform/app_identity/gae_app_identity_services.py @@ -20,6 +20,8 @@ import os +from typing import Optional + _GCS_RESOURCE_BUCKET_NAME_SUFFIX = '-resources' @@ -43,7 +45,7 @@ def get_application_id() -> str: return oppia_project_id -def get_gcs_resource_bucket_name() -> str: +def get_gcs_resource_bucket_name(oppia_project_id: Optional[str] = None) -> str: """Returns the application's bucket name for GCS resources, which depends on the application ID in production mode, or default bucket name in development mode. @@ -55,7 +57,13 @@ def get_gcs_resource_bucket_name() -> str: if we try to use it in production mode but the default bucket hasn't been enabled through the project console. + Args: + oppia_project_id: Optional[str]. The Google Cloud Project ID. Explicitly + required when running on Beam Dataflow, as workers cannot + retrieve the ID from environment variables. + Returns: str. The bucket name for the application's GCS resources. """ - return '%s%s' % (get_application_id(), _GCS_RESOURCE_BUCKET_NAME_SUFFIX) + project_id = oppia_project_id or get_application_id() + return '%s%s' % (project_id, _GCS_RESOURCE_BUCKET_NAME_SUFFIX) diff --git a/core/platform/search/elastic_search_services.py b/core/platform/search/elastic_search_services.py index 8907a667848d6..d99427b6bd64e 100644 --- a/core/platform/search/elastic_search_services.py +++ b/core/platform/search/elastic_search_services.py @@ -413,6 +413,8 @@ def blog_post_summaries_search( 'multi_match': { 'query': query_string, 'fields': ['title', 'summary'], + 'type': 'bool_prefix', + 'operator': 'and', } } ] diff --git a/core/platform/search/elastic_search_services_test.py b/core/platform/search/elastic_search_services_test.py index c0af1f204b71c..fd0dce3f708d4 100644 --- a/core/platform/search/elastic_search_services_test.py +++ b/core/platform/search/elastic_search_services_test.py @@ -390,6 +390,8 @@ def mock_search( 'multi_match': { 'query': 'query', 'fields': ['title', 'summary'], + 'type': 'bool_prefix', + 'operator': 'and', } } ], diff --git a/core/platform/secrets/cloud_secrets_services.py b/core/platform/secrets/cloud_secrets_services.py index b8c15fa650e78..28a6a1bedb510 100644 --- a/core/platform/secrets/cloud_secrets_services.py +++ b/core/platform/secrets/cloud_secrets_services.py @@ -45,16 +45,20 @@ @functools.lru_cache(maxsize=64) -def get_secret(name: str) -> Optional[str]: +def get_secret(name: str, project_id: Optional[str] = None) -> Optional[str]: """Gets the value of a secret. Args: name: str. The name of the secret to retrieve. + project_id: Optional[str]. The Google Cloud Project ID. Explicitly + required when running on Beam Dataflow, as workers cannot + retrieve the ID from environment variables. Returns: str. The value of the secret. """ - oppia_project_id = app_identity_services.get_application_id() + oppia_project_id = project_id or app_identity_services.get_application_id() + secret_name = f'projects/{oppia_project_id}/secrets/{name}/versions/latest' try: response = CLIENT.access_secret_version(request={'name': secret_name}) diff --git a/core/platform/secrets/cloud_secrets_services_test.py b/core/platform/secrets/cloud_secrets_services_test.py index 1f3f571afd91a..7d43991982fc0 100644 --- a/core/platform/secrets/cloud_secrets_services_test.py +++ b/core/platform/secrets/cloud_secrets_services_test.py @@ -35,6 +35,19 @@ def test_get_secret_returns_existing_secret(self) -> None: ): self.assertEqual(cloud_secrets_services.get_secret('name'), 'secre') + def test_get_secret_with_oppia_project_id_as_parameter(self) -> None: + with self.swap_to_always_return( + cloud_secrets_services.CLIENT, + 'access_secret_version', + types.SimpleNamespace(payload=types.SimpleNamespace(data=b'secre')), + ): + self.assertEqual( + cloud_secrets_services.get_secret( + 'name', project_id='project-id' + ), + 'secre', + ) + def test_get_secret_returns_none_when_secret_does_not_exist(self) -> None: with self.swap_to_always_raise( cloud_secrets_services.CLIENT, diff --git a/core/platform/secrets/dev_mode_secrets_services.py b/core/platform/secrets/dev_mode_secrets_services.py index 09482ae87d47c..92225758e4353 100644 --- a/core/platform/secrets/dev_mode_secrets_services.py +++ b/core/platform/secrets/dev_mode_secrets_services.py @@ -26,11 +26,14 @@ @functools.lru_cache(maxsize=64) -def get_secret(name: str) -> Optional[str]: +def get_secret(name: str, _: Optional[str] = None) -> Optional[str]: """Gets the value of a secret. This is only dev mode version of the secrets. Args: name: str. The name of the secret to retrieve. + _: Optional[str]. The Google Cloud Project ID. Explicitly + required when running on Beam Dataflow, as workers cannot + retrieve the ID from environment variables. Returns: str. The value of the secret. diff --git a/core/platform/speech_synthesis/azure_speech_synthesis_services.py b/core/platform/speech_synthesis/azure_speech_synthesis_services.py index f1dc7c9bd8668..3774922e5b503 100644 --- a/core/platform/speech_synthesis/azure_speech_synthesis_services.py +++ b/core/platform/speech_synthesis/azure_speech_synthesis_services.py @@ -21,12 +21,12 @@ from __future__ import annotations -import json -import os +import logging import re +import time +from xml.sax import saxutils -from core import feconf -from core.constants import constants +from core import constants, feconf from core.domain import voiceover_services from core.platform import models @@ -61,6 +61,8 @@ """ +MAX_RETRIES_FOR_VOICEOVER_SYNTHESIS_WITH_EXPONENTIAL_BACKOFF = 10 + class WordBoundaryCollection: """This class handles word boundary events to collect the time offsets @@ -103,11 +105,9 @@ def get_azure_voicecode_from_language_accent_code( str. The Azure voice code associated with the given language accent code. """ - file_path = os.path.join( - feconf.VOICEOVERS_DATA_DIR, 'autogeneratable_language_accent_list.json' + autogeneratable_language_accent_list: Dict[str, Dict[str, str]] = ( + constants.autogeneratable_language_accent_constants ) - with open(file_path, 'r', encoding='utf-8') as f: - autogeneratable_language_accent_list = json.loads(f.read()) voice_code: str = autogeneratable_language_accent_list[ language_accent_code @@ -130,7 +130,7 @@ def process_factorial_in_text( str. The processed text with factorial expressions replaced by their corresponding words or phrases. """ - pronounciation = math_symbol_pronunciations['!'] + ' ' + pronounciation = math_symbol_pronunciations.get('!', '') + ' ' return re.sub(r'(\d+)!', pronounciation + r'\1', text) @@ -254,18 +254,29 @@ def convert_plaintext_to_ssml_content( ) math_symbol_pronunciations = ( - constants.LANGUAGE_CODE_TO_MATH_SYMBOL_PRONUNCIATIONS.get( + constants.constants.LANGUAGE_CODE_TO_MATH_SYMBOL_PRONUNCIATIONS.get( language_code, {} ) ) main_ssml_content = '' for content in content_list: + # Escaping special characters in the content to ensure they are + # pronounced correctly by the Azure Text-to-Speech service. + # This includes characters like <, >, &, etc. + content = saxutils.escape(content) # Updates the content to pronounce `-` correctly in the given language. if ' - ' in content: - content = content.replace( - '-', MATH_TEMPLATE_SSML_BLOCK % math_symbol_pronunciations['-'] - ) + pattern = re.compile(r'(\d+)\s*-\s*(\d+)') + + def replacer(match: re.Match[str]) -> str: + num1, num2 = match.groups() + pronunciation = ( + MATH_TEMPLATE_SSML_BLOCK % math_symbol_pronunciations['-'] + ) + return '%s %s %s' % (num1, pronunciation, num2) + + content = pattern.sub(replacer, content) # Update the content to pronounce `*` correctly in the given language. if ' * ' in content: @@ -286,7 +297,8 @@ def convert_plaintext_to_ssml_content( # Update the content to pronounce `/` correctly in the given language. if ' / ' in content: content = content.replace( - '/', MATH_TEMPLATE_SSML_BLOCK % math_symbol_pronunciations['÷'] + ' / ', + MATH_TEMPLATE_SSML_BLOCK % math_symbol_pronunciations['÷'], ) # Update the content to pronounce `÷` correctly in the given language. @@ -332,8 +344,10 @@ def convert_plaintext_to_ssml_content( def regenerate_speech_from_text( - plaintext: str, language_accent_code: str -) -> Tuple[bytes, List[Dict[str, Union[str, float]]], Optional[str]]: + plaintext: str, + language_accent_code: str, + oppia_project_id: Optional[str] = None, +) -> Tuple[Optional[bytes], List[Dict[str, Union[str, float]]], Optional[str]]: """Regenerates speech (Oppia's voiceovers) from the provided text. This method uses Azure Text-to-Speech to synthesize speech from the input @@ -344,6 +358,9 @@ def regenerate_speech_from_text( plaintext: str. The plaintext that needs to be synthesized into speech. language_accent_code: str. The language accent code in which the speech is to be synthesized. + oppia_project_id: Optional[str]. The Google Cloud Project ID. Explicitly + required when running on Beam Dataflow, as workers cannot + retrieve the ID from environment variables. Returns: tuple. A tuple containing three elements: @@ -360,7 +377,9 @@ def regenerate_speech_from_text( """ # Azure text-to-speech API key. - azure_tts_api_key = secrets_services.get_secret('AZURE_TTS_API_KEY') + azure_tts_api_key = secrets_services.get_secret( + 'AZURE_TTS_API_KEY', oppia_project_id + ) if azure_tts_api_key is None: raise Exception('Azure TTS API key is not available.') @@ -393,18 +412,85 @@ def regenerate_speech_from_text( plaintext, language_accent_code ) - speech_synthesis_result = speech_synthesizer.speak_ssml_async( - ssml_text_for_speech_synthesis - ).get() + delay_in_sec_before_retrying = 1 + binary_audio_data = None + error_details = None + + for _ in range( + MAX_RETRIES_FOR_VOICEOVER_SYNTHESIS_WITH_EXPONENTIAL_BACKOFF + ): + logging.info( + 'Voiceover synthesis log: Retrying speech synthesis after %s seconds delay.', + delay_in_sec_before_retrying, + ) + time.sleep(delay_in_sec_before_retrying) + + speech_synthesis_result = speech_synthesizer.speak_ssml_async( + ssml_text_for_speech_synthesis + ).get() + + if ( + speech_synthesis_result.reason + == speechsdk.ResultReason.SynthesizingAudioCompleted + ): + binary_audio_data = speech_synthesis_result.audio_data + error_details = None + break - binary_audio_data = speech_synthesis_result.audio_data + if speech_synthesis_result.reason == speechsdk.ResultReason.Canceled: + cancellation_details = speech_synthesis_result.cancellation_details + + if ( + cancellation_details.reason + == speechsdk.CancellationReason.Error + ): + error_details = cancellation_details.error_details + error_code = cancellation_details.error_code + + logging.error( + 'Voiceover synthesis log: Speech synthesis failed for content %s with error code %s and details: %s' + % (plaintext, error_code, error_details) + ) + + # Exponential backoff for retrying speech synthesis in case of too + # many requests, connection failure, or service timeout errors. + if error_code in [ + speechsdk.CancellationErrorCode.TooManyRequests, + speechsdk.CancellationErrorCode.ConnectionFailure, + speechsdk.CancellationErrorCode.ServiceTimeout, + ]: + logging.info( + 'Voiceover synthesis log: Known error encountered, retrying with exponential backoff.' + ) + delay_in_sec_before_retrying *= 2 + continue + + logging.info( + 'Voiceover synthesis log: Non-retryable error encountered, aborting further attempts.' + ) + break + + error_details = ( + 'Speech synthesis was canceled for reason: %s' + % cancellation_details.reason + ) + logging.error( + 'Voiceover synthesis log: Voiceover synthesis error: %s for content: %s' + % (error_details, plaintext) + ) + break - error_details = None - if speech_synthesis_result.reason == speechsdk.ResultReason.Canceled: - cancellation_details = speech_synthesis_result.cancellation_details + error_details = ( + 'Speech synthesis failed for reason: %s' + % speech_synthesis_result.reason + ) + logging.error( + 'Voiceover synthesis log: Voiceover synthesis error: %s for content: %s' + % (error_details, plaintext) + ) + break - if cancellation_details.reason == speechsdk.CancellationReason.Error: - error_details = cancellation_details.error_details + logging.info('Voiceover synthesis log: Speech synthesis attempt completed.') return ( binary_audio_data, diff --git a/core/platform/speech_synthesis/azure_speech_synthesis_services_test.py b/core/platform/speech_synthesis/azure_speech_synthesis_services_test.py index 2f24a025b02ad..b84e8b65dc84d 100644 --- a/core/platform/speech_synthesis/azure_speech_synthesis_services_test.py +++ b/core/platform/speech_synthesis/azure_speech_synthesis_services_test.py @@ -42,9 +42,9 @@ def setUp(self) -> None: self.swap_api_key_secrets_return_secret = self.swap_with_checks( secrets_services, 'get_secret', - lambda _: 'azure_key', + lambda _, oppia_project_id: 'azure_key', expected_args=[ - ('AZURE_TTS_API_KEY',), + ('AZURE_TTS_API_KEY', None), ], ) @@ -200,7 +200,7 @@ def test_regenerate_speech_from_text_failed_for_invalid_credentials( plaintext = 'This is a test text' language_accent_code = 'en-US' - mock_audio_data = b'' + mock_audio_data = None mock_word_boundaries: List[Dict[str, Union[str, float]]] = [] mock_error_details = ( 'WebSocket upgrade failed: Authentication error (401). ' @@ -238,7 +238,7 @@ def test_regenerate_speech_from_text_failed( plaintext, language_accent_code ) ) - mock_audio_data = b'' + mock_audio_data = None mock_speech_config_instance = mock_speech_config.return_value mock_speech_config_instance.set_speech_synthesis_output_format = ( @@ -289,6 +289,168 @@ def test_regenerate_speech_from_text_failed( self.assertEqual(result_audio_offsets, mock_word_boundaries) self.assertEqual(result_error, error_details) + @mock.patch('azure.cognitiveservices.speech.SpeechSynthesizer') + @mock.patch('azure.cognitiveservices.speech.SpeechConfig') + @mock.patch( + 'core.platform.speech_synthesis.' + 'azure_speech_synthesis_services.WordBoundaryCollection' + ) + def test_regenerate_speech_from_text_failed_due_to_multiple_requests( + self, + mock_word_boundary_collection: mock.Mock, + mock_speech_config: mock.Mock, + mock_speech_synthesizer: mock.Mock, + ) -> None: + plaintext = 'This is a test text' + language_accent_code = 'en-US' + mock_audio_data = None + + mock_speech_config_instance = mock_speech_config.return_value + mock_speech_config_instance.set_speech_synthesis_output_format = ( + mock.MagicMock() + ) + mock_speech_synthesizer_instance = mock_speech_synthesizer.return_value + mock_speech_synthesis_result = mock.MagicMock() + mock_speech_synthesis_result.audio_data = mock_audio_data + mock_cancellation_details = mock.MagicMock() + + error_details = ( + 'WebSocket upgrade failed: Too many requests (429). Please check ' + 'subscription information and region name. USP state: Sending. ' + 'Received audio size: 0 bytes' + ) + mock_cancellation_details.reason = speechsdk.CancellationReason.Error + mock_cancellation_details.error_details = error_details + mock_cancellation_details.error_code = ( + speechsdk.CancellationErrorCode.TooManyRequests + ) + + mock_speech_synthesis_result.reason = speechsdk.ResultReason.Canceled + mock_speech_synthesis_result.cancellation_details = ( + mock_cancellation_details + ) + ( + mock_speech_synthesizer_instance.speak_ssml_async.return_value.get.return_value + ) = mock_speech_synthesis_result + mock_word_boundary_instance = mock.MagicMock() + mock_word_boundaries: List[Dict[str, Union[str, float]]] = [] + mock_word_boundary_instance.audio_offset_list = mock_word_boundaries + mock_word_boundary_collection.return_value = mock_word_boundary_instance + + with self.swap_api_key_secrets_return_secret: + result_binary_data, result_audio_offsets, result_error = ( + azure_speech_synthesis_services.regenerate_speech_from_text( + plaintext, language_accent_code + ) + ) + + self.assertEqual(result_binary_data, mock_audio_data) + self.assertEqual(result_audio_offsets, mock_word_boundaries) + self.assertEqual(result_error, error_details) + + @mock.patch('azure.cognitiveservices.speech.SpeechSynthesizer') + @mock.patch('azure.cognitiveservices.speech.SpeechConfig') + @mock.patch( + 'core.platform.speech_synthesis.' + 'azure_speech_synthesis_services.WordBoundaryCollection' + ) + def test_regenerate_speech_from_text_failed_due_to_end_of_stream( + self, + mock_word_boundary_collection: mock.Mock, + mock_speech_config: mock.Mock, + mock_speech_synthesizer: mock.Mock, + ) -> None: + plaintext = 'This is a test text' + language_accent_code = 'en-US' + mock_audio_data = None + + mock_speech_config_instance = mock_speech_config.return_value + mock_speech_config_instance.set_speech_synthesis_output_format = ( + mock.MagicMock() + ) + mock_speech_synthesizer_instance = mock_speech_synthesizer.return_value + mock_speech_synthesis_result = mock.MagicMock() + mock_speech_synthesis_result.audio_data = mock_audio_data + mock_cancellation_details = mock.MagicMock() + + error_details = 'Speech synthesis was canceled for reason: CancellationReason.EndOfStream' + mock_cancellation_details.reason = ( + speechsdk.CancellationReason.EndOfStream + ) + + mock_speech_synthesis_result.reason = speechsdk.ResultReason.Canceled + mock_speech_synthesis_result.cancellation_details = ( + mock_cancellation_details + ) + ( + mock_speech_synthesizer_instance.speak_ssml_async.return_value.get.return_value + ) = mock_speech_synthesis_result + mock_word_boundary_instance = mock.MagicMock() + mock_word_boundaries: List[Dict[str, Union[str, float]]] = [] + mock_word_boundary_instance.audio_offset_list = mock_word_boundaries + mock_word_boundary_collection.return_value = mock_word_boundary_instance + + with self.swap_api_key_secrets_return_secret: + result_binary_data, result_audio_offsets, result_error = ( + azure_speech_synthesis_services.regenerate_speech_from_text( + plaintext, language_accent_code + ) + ) + + self.assertEqual(result_binary_data, mock_audio_data) + self.assertEqual(result_audio_offsets, mock_word_boundaries) + self.assertEqual(result_error, error_details) + + @mock.patch('azure.cognitiveservices.speech.SpeechSynthesizer') + @mock.patch('azure.cognitiveservices.speech.SpeechConfig') + @mock.patch( + 'core.platform.speech_synthesis.' + 'azure_speech_synthesis_services.WordBoundaryCollection' + ) + def test_regenerate_speech_from_text_failed_due_to_unknown_reason( + self, + mock_word_boundary_collection: mock.Mock, + mock_speech_config: mock.Mock, + mock_speech_synthesizer: mock.Mock, + ) -> None: + plaintext = 'This is a test text' + language_accent_code = 'en-US' + mock_audio_data = None + + mock_speech_config_instance = mock_speech_config.return_value + mock_speech_config_instance.set_speech_synthesis_output_format = ( + mock.MagicMock() + ) + mock_speech_synthesizer_instance = mock_speech_synthesizer.return_value + mock_speech_synthesis_result = mock.MagicMock() + mock_speech_synthesis_result.audio_data = mock_audio_data + mock_cancellation_details = mock.MagicMock() + + error_details = ( + 'Speech synthesis failed for reason: UnknownCancellationReason' + ) + mock_cancellation_details.reason = 'UnknownCancellationReason' + + mock_speech_synthesis_result.reason = 'UnknownCancellationReason' + ( + mock_speech_synthesizer_instance.speak_ssml_async.return_value.get.return_value + ) = mock_speech_synthesis_result + mock_word_boundary_instance = mock.MagicMock() + mock_word_boundaries: List[Dict[str, Union[str, float]]] = [] + mock_word_boundary_instance.audio_offset_list = mock_word_boundaries + mock_word_boundary_collection.return_value = mock_word_boundary_instance + + with self.swap_api_key_secrets_return_secret: + result_binary_data, result_audio_offsets, result_error = ( + azure_speech_synthesis_services.regenerate_speech_from_text( + plaintext, language_accent_code + ) + ) + + self.assertEqual(result_binary_data, mock_audio_data) + self.assertEqual(result_audio_offsets, mock_word_boundaries) + self.assertEqual(result_error, error_details) + def test_should_return_word_boundary_collection_correctly(self) -> None: word_boundary_collection = ( azure_speech_synthesis_services.WordBoundaryCollection() @@ -400,6 +562,19 @@ def test_should_convert_plaintext_to_ssml_content_correctly(self) -> None: self._get_ssml_content(expected_main_content, language_accent_code), ) + plaintext = 'Hello - welcome to Oppia!' + expected_main_content = 'Hello - welcome to Oppia!' + + ssml_content = ( + azure_speech_synthesis_services.convert_plaintext_to_ssml_content( + plaintext, language_accent_code + ) + ) + self.assertEqual( + ssml_content, + self._get_ssml_content(expected_main_content, language_accent_code), + ) + plaintext = 'Find the value of 5 + 3.' expected_main_content = ( 'Find the value of 5
+
+
diff --git a/core/templates/components/campaign-banner/campaign-banner.component.spec.ts b/core/templates/components/campaign-banner/campaign-banner.component.spec.ts
new file mode 100644
index 0000000000000..335bb4281c04f
--- /dev/null
+++ b/core/templates/components/campaign-banner/campaign-banner.component.spec.ts
@@ -0,0 +1,323 @@
+// Copyright 2026 The Oppia Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS-IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @fileoverview Unit tests for CampaignBannerComponent.
+ */
+
+import {ComponentFixture, TestBed} from '@angular/core/testing';
+import {HttpClientTestingModule} from '@angular/common/http/testing';
+
+import {CampaignBannerComponent} from './campaign-banner.component';
+import {UrlInterpolationService} from 'domain/utilities/url-interpolation.service';
+import {PlatformFeatureService} from 'services/platform-feature.service';
+import {AppConstants} from 'app.constants';
+import {SiteAnalyticsService} from 'services/site-analytics.service';
+import {WindowRef} from 'services/contextual/window-ref.service';
+
+interface CampaignConfig {
+ startDate: Date;
+ endDate: Date;
+ bannerReRenderIntervalMs: number;
+ bannerImageRelativePath: string;
+}
+
+class MockUrlInterpolationService {
+ getStaticImageUrl(imagePath: string): string {
+ return `/assets/${imagePath}`;
+ }
+}
+
+class MockPlatformFeatureService {
+ status = {
+ EnableCampaignBanner: {
+ isEnabled: true,
+ },
+ EnableCampaignBannerTestMode: {
+ isEnabled: false,
+ },
+ };
+}
+
+class MockWindowRef {
+ nativeWindow = {
+ location: {
+ pathname: '/learn/math',
+ href: '',
+ },
+ gtag: () => {},
+ };
+}
+
+class MockSiteAnalyticsService {
+ registerCampaignBannerDonateButtonClick(): void {}
+ registerCampaignBannerVisibility(): void {}
+}
+
+describe('CampaignBannerComponent', () => {
+ let component: CampaignBannerComponent;
+ let fixture: ComponentFixture
+
+
+
+
+
+
+
+
+
+
+
+ + Most children graduate school without ever learning + how money works. +
+ +
+
+ + This {{ campaignEndMonth }}, Oppia is launching free, interactive financial literacy + lessons for learners who need them most. Your donation helps us + reach more of them. +
+ +
+
+
+
+
+ 50%+
+
+ Improvement in Learning Outcome
+ + Your donation helps us reach more learners and empower the next + generation with essential financial skills. +
+
+
+ Donate Now
+
+
+
+
+ Campaign ends {{ campaignEndMonth }} {{ campaignEndDay }}
+
+
+ ![Campaign image]()
+
+