diff --git a/modules/weko-logging/weko_logging/models.py b/modules/weko-logging/weko_logging/models.py index 58e2d5c749..0e88a918fb 100644 --- a/modules/weko-logging/weko_logging/models.py +++ b/modules/weko-logging/weko_logging/models.py @@ -22,11 +22,9 @@ group_id_seq = Sequence('user_activity_log_group_id_seq', metadata=db.metadata) -class _UserActivityLogBase(db.Model): +class _UserActivityLogBase: """User activity log model.""" - __tablename__ = 'user_activity_logs' - id = db.Column( db.Integer(), primary_key=True, diff --git a/test/tavern/create_item_type_template.py b/test/tavern/create_item_type_template.py index e904a9534b..a7cf71db2f 100644 --- a/test/tavern/create_item_type_template.py +++ b/test/tavern/create_item_type_template.py @@ -35,14 +35,14 @@ def create_itemtype_template(): # Create template for entering data template = create_template(result['schema']) template['$schema'] = '/items/jsonschema/' + str(schema[0]) - template['shared_user_id'] = -1 + template['shared_user_ids'] = [] with open('request_params/item_type_template/template/' + file_name + '.json', 'w', encoding='utf-8') as f: f.write(json.dumps(template, indent=4, ensure_ascii=False)) # Create template for generating random data random_template, _ = create_random_template(result['schema']) random_template['$schema'] = '/items/jsonschema/' + str(schema[0]) - random_template['shared_user_id'] = -1 + random_template['shared_user_ids'] = [] with open('request_params/item_type_template/template/' + file_name + '_random.json', 'w', encoding='utf-8') as f: f.write(json.dumps(random_template, indent=4, ensure_ascii=False)) @@ -163,9 +163,10 @@ def create_template(json): text_type = ['text', 'textarea', 'select', 'radios'] result = {} for k, v in json.items(): - if v.get('title') == 'Identifier Registration'\ - or v.get('title').startswith('Persistent Identifier')\ - or v.get('title') == 'File Information': + if v.get('title') \ + and (v.get('title') == 'Identifier Registration'\ + or v.get('title').startswith('Persistent Identifier')\ + or v.get('title') == 'File Information'): continue if v.get('format'): if v.get('format') == 'datetime': @@ -210,7 +211,10 @@ def create_random_template(json): result = {} has_uri = False for k, v in json.items(): - if v.get('title') == 'Identifier Registration' or v.get('title').startswith('Persistent Identifier') or v.get('title') == 'File Information': + if v.get('title') \ + and (v.get('title') == 'Identifier Registration' \ + or v.get('title').startswith('Persistent Identifier') \ + or v.get('title') == 'File Information'): continue if v.get('format'): if v.get('format') == 'datetime': diff --git a/test/tavern/docker-compose.yml b/test/tavern/docker-compose.yml index 926190df32..1d1d64b8ce 100644 --- a/test/tavern/docker-compose.yml +++ b/test/tavern/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.2' - services: tavern: build: @@ -8,7 +6,7 @@ services: environment: - TZ=Asia/Tokyo - OPENSEARCH_INITIAL_ADMIN_PASSWORD=WekoOpensearch123! - volumes: + volumes: - type: bind source: . target: /tavern diff --git a/test/tavern/generator/json/generate_index_data.py b/test/tavern/generator/json/generate_index_data.py new file mode 100644 index 0000000000..aee8c2d6a4 --- /dev/null +++ b/test/tavern/generator/json/generate_index_data.py @@ -0,0 +1,491 @@ +import argparse +import copy +import json +import os +import random +import string +import sys +from datetime import datetime, timedelta +from contextlib import closing + +sys.path.append(os.path.abspath(os.getcwd())) + +from helper.common.connect_helper import connect_db + +EMPTY_TARGET_COLUMNS = ["index_name", "index_name_english"] + +class Group: + def __init__(self, id, name): + """Initialize a Group instance. + + Args: + id(int): Group ID. + name(str): Group name. + """ + self.id = id + self.name = name + + def to_dict(self): + """Convert the Group instance to a dictionary. + + Returns: + dict: Dictionary representation of the Group. + """ + return { + "id": self.id, + "name": self.name + } + + +class Role: + def __init__(self, id, name): + """Initialize a Role instance. + + Args: + id(int): Role ID. + name(str): Role name. + """ + self.id = id + self.name = name + + def to_dict(self): + """Convert the Role instance to a dictionary. + + Returns: + dict: Dictionary representation of the Role. + """ + return { + "id": self.id, + "name": self.name + } + + +class Index: + def __init__(self, id, parent, position): + """Initialize an Index instance. + + Args: + id(int): Index ID. + parent(int): Parent index ID. + position(int): Position of the index. + """ + self.id = id + self.parent = parent + self.position = position + + def to_dict(self): + """Convert the Index instance to a dictionary. + + Returns: + dict: Dictionary representation of the Index. + """ + return { + "id": self.id, + "parent": self.parent, + "position": self.position + } + + +def get_groups(): + """Retrieve groups from the database. + + Returns: + list[Group]: List of Group instances. + """ + groups = [] + query = "SELECT id, name FROM accounts_group;" + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cur: + cur.execute(query) + result = cur.fetchall() + for g in result: + groups.append(Group(g[0], g[1])) + return groups + + +def get_roles(): + """Retrieve roles from the database. + + Returns: + list[Role]: List of Role instances. + """ + roles = [] + query = "SELECT id, name FROM accounts_role;" + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cur: + cur.execute(query) + result = cur.fetchall() + for r in result: + roles.append(Role(r[0], r[1])) + + # Add predefined roles + roles.append(Role(-98, "Authenticated User")) + roles.append(Role(-99, "Guest")) + return roles + + +def get_indices(): + """Retrieve indices from the database. + + Returns: + list[Index]: List of Index instances. + """ + indices = [] + query = "SELECT id, parent, position FROM index;" + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cur: + cur.execute(query) + result = cur.fetchall() + for i in result: + indices.append(Index(i[0], i[1], i[2])) + return indices + + +def set_groups_to_template(groups): + """Assign groups to allow or deny lists in the template. + + Args: + groups(list[Group]): List of Group instances. + + Returns: + dict: Dictionary with allow and deny lists. + """ + set_value = { + "allow": [], + "deny": [] + } + for group in groups: + if random.choice([True, False]): + set_value["allow"].append(group.to_dict()) + else: + set_value["deny"].append(group.to_dict()) + return set_value + + +def set_roles_to_template(roles): + """Assign roles to allow or deny lists in the template. + + Args: + roles(list[Role]): List of Role instances. + + Returns: + dict: Dictionary with allow and deny lists. + """ + super_role = ["System Administrator", "Repository Administrator"] + set_value = { + "allow": [], + "deny": [] + } + for role in roles: + if role.name in super_role: + continue + if random.choice([True, False]): + set_value["allow"].append(role.to_dict()) + else: + set_value["deny"].append(role.to_dict()) + return set_value + + +def set_index_to_template(indices, is_community, recursive): + """Select an index for the template. + + Args: + indices(list[Index]): List of Index instances. + recursive(bool): Whether to consider recursive indices. + is_community(bool): Whether the index is for a community. + + Returns: + dict: Dictionary representation of the selected index. + """ + target_indices = [] + community_indices = [] + no_community_indices = [] + for index in indices: + if index.id == 1: + community_indices.append(index) + else: + community_indices_ids = [i.id for i in community_indices] + if index.parent in community_indices_ids: + community_indices.append(index) + else: + no_community_indices.append(index) + + loop_indices = [] + if is_community is None: + loop_indices = indices + elif is_community: + loop_indices = community_indices + else: + loop_indices = no_community_indices + + if recursive: + for index in loop_indices: + if [i for i in indices if i.parent == index.id]: + target_indices.append(index) + else: + target_indices = loop_indices + target_index = random.choice(target_indices) + return { + "id": target_index.id, + "parent": target_index.parent, + "position": target_index.position + } + +def set_gakunin_groups(browsing_role, contribute_role): + """Separate Gakunin groups from roles. + + Args: + browsing_role(dict): Browsing role data. + contribute_role(dict): Contribute role data. + + Returns: + dict: Dictionary with separated Gakunin groups. + """ + def is_gakunin_group(role_name): + """Check if the role name corresponds to a Gakunin group. + + Args: + role_name(str): Name of the role. + + Returns: + bool: True if it is a Gakunin group, False otherwise. + """ + return role_name.startswith("jc_") and role_name.find("groups") != -1 + + allow_browsing_role = [] + deny_browsing_role = [] + allow_contribute_role = [] + deny_contribute_role = [] + allow_browsing_group = [] + deny_browsing_group = [] + allow_contribute_group = [] + deny_contribute_group = [] + for role in browsing_role["allow"]: + if is_gakunin_group(role["name"]): + allow_browsing_group.append({ + "id": str(role["id"]) + "gr", + "name": role["name"] + }) + else: + allow_browsing_role.append(role) + for role in browsing_role["deny"]: + if is_gakunin_group(role["name"]): + deny_browsing_group.append({ + "id": str(role["id"]) + "gr", + "name": role["name"] + }) + else: + deny_browsing_role.append(role) + for role in contribute_role["allow"]: + if is_gakunin_group(role["name"]): + allow_contribute_group.append({ + "id": str(role["id"]) + "gr", + "name": role["name"] + }) + else: + allow_contribute_role.append(role) + for role in contribute_role["deny"]: + if is_gakunin_group(role["name"]): + deny_contribute_group.append({ + "id": str(role["id"]) + "gr", + "name": role["name"] + }) + else: + deny_contribute_role.append(role) + return { + "browsing_group": { + "allow": allow_browsing_group, + "deny": deny_browsing_group + }, + "contribute_group": { + "allow": allow_contribute_group, + "deny": deny_contribute_group + }, + "browsing_role": { + "allow": allow_browsing_role, + "deny": deny_browsing_role + }, + "contribute_role": { + "allow": allow_contribute_role, + "deny": deny_contribute_role + } + } + + +def generate_index_data(indices, is_community=None, recursive = False): + """Generate index data based on the template. + + Args: + indices(list[Index]): List of Index instances. + is_community(bool, optional): Whether to generate data for community index. Defaults to None. + recursive(bool, optional): Whether to generate recursive index data. Defaults to False. + + Returns: + dict: Generated index data. + """ + file = "generator/json/template_file/index_template.json" + generated_data = {} + groups = get_groups() + roles = get_roles() + with open(file, "r") as f: + data = json.load(f) + for k, v in data.items(): + if v == "str": + length = random.randint(5, 15) + generated_data[k] = "".join(random.choices(string.ascii_letters, k=length)) + elif v == "int": + generated_data[k] = random.randint(1, 100) + elif v == "bool": + generated_data[k] = random.choice([True, False]) + elif v == "date": + today = datetime.now() + one_month_ago = today - timedelta(days=30) + one_month_later = today + timedelta(days=30) + random_date = one_month_ago + (one_month_later - one_month_ago) * random.random() + generated_data[k] = random_date.strftime("%Y%m%d") + elif v == "Group": + generated_data[k] = set_groups_to_template(groups) + elif v == "Role": + generated_data[k] = set_roles_to_template(roles) + + generated_data.update(set_index_to_template(indices, is_community, recursive)) + generated_data.update(set_gakunin_groups( + generated_data["browsing_role"], + generated_data["contribute_role"] + )) + + return generated_data + + +def main(start_index, end_index): + """Main function to generate index data JSON files. + + Args: + start_index(int): Starting index ID for data generation. + end_index(int): Ending index ID for data generation. + """ + index_data = { + "biblio_flag": False, + "image_name": "", + "is_deleted": False, + "recursive_browsing_group": False, + "recursive_browsing_role": False, + "recursive_contribute_group": False, + "recursive_contribute_role": False, + "recursive_coverpage_check": False, + "recursive_public_state": False, + "thumbnail_delete_flag": False + } + output_folder = "request_params/index/test_data" + indices = get_indices() + users = { + "sysadmin": 1, + "repoadmin": 2, + "comadmin": 5, + "contributor": 3, + "user": 4, + "guest": 0 + } + for user, user_id in users.items(): + if not os.path.exists(f"{output_folder}/{user}"): + os.makedirs(f"{output_folder}/{user}") + + if start_index is not None and end_index is not None: + indices = [index for index in indices if start_index <= index.id <= end_index] + for index in indices: + generated_data = copy.deepcopy(index_data) + generated_data.update(generate_index_data([index])) + generated_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/generated_index_data_{index.id}.json", "w") as f: + json.dump(generated_data, f, indent=4) + else: + for index in indices: + generated_data = copy.deepcopy(index_data) + generated_data.update(generate_index_data([index])) + generated_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_{index.id}.json", "w") as f: + json.dump(generated_data, f, indent=4) + + recursive_key = [k for k in index_data.keys() if k.startswith("recursive_")] + ["biblio_flag"] + if user in ["comadmin", "contributor"]: + for key in recursive_key: + generated_recursive_data = copy.deepcopy(index_data) + generated_recursive_data[key] = True + generated_recursive_data.update( + generate_index_data(indices, is_community=True, recursive=True)) + generated_recursive_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_{key}_in_community.json", "w") as f: + json.dump(generated_recursive_data, f, indent=4) + + generated_recursive_data = copy.deepcopy(index_data) + generated_recursive_data[key] = True + generated_recursive_data.update( + generate_index_data(indices, is_community=False, recursive=True)) + generated_recursive_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_{key}_out_community.json", "w") as f: + json.dump(generated_recursive_data, f, indent=4) + + for col in EMPTY_TARGET_COLUMNS: + # Generate data with empty string for the target column + generated_empty_data = copy.deepcopy(index_data) + generated_empty_data.update(generate_index_data(indices, is_community=True)) + generated_empty_data[col] = "" + generated_empty_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_empty_{col}_in_community.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + generated_empty_data[col] = "".join(random.choices(string.ascii_letters, k=1)) + with open(f"{output_folder}/{user}/index_data_1char_{col}_in_community.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + generated_empty_data = copy.deepcopy(index_data) + generated_empty_data.update(generate_index_data(indices, is_community=False)) + generated_empty_data[col] = "" + generated_empty_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_empty_{col}_out_community.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + generated_empty_data[col] = "".join(random.choices(string.ascii_letters, k=1)) + with open(f"{output_folder}/{user}/index_data_1char_{col}_out_community.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + else: + for key in recursive_key: + generated_recursive_data = copy.deepcopy(index_data) + generated_recursive_data[key] = True + generated_recursive_data.update(generate_index_data(indices, recursive=True)) + generated_recursive_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_{key}.json", "w") as f: + json.dump(generated_recursive_data, f, indent=4) + + for col in EMPTY_TARGET_COLUMNS: + # Generate data with empty string for the target column + generated_empty_data = copy.deepcopy(index_data) + generated_empty_data.update(generate_index_data(indices)) + generated_empty_data[col] = "" + generated_empty_data["owner_user_id"] = user_id + with open(f"{output_folder}/{user}/index_data_empty_{col}.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + generated_empty_data[col] = "".join(random.choices(string.ascii_letters, k=1)) + with open(f"{output_folder}/{user}/index_data_1char_{col}.json", "w") as f: + json.dump(generated_empty_data, f, indent=4) + + +def parse_args(): + """Parse command-line arguments. + + Returns: + argparse.Namespace: Parsed arguments. + """ + parser = argparse.ArgumentParser(description="Generate index data JSON.") + parser.add_argument("-s", "--start", type=int, help="Starting index ID") + parser.add_argument("-e", "--end", type=int, help="Ending index ID") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + start_index = args.start + end_index = args.end + main(start_index, end_index) diff --git a/test/tavern/generator/json/generate_json.py b/test/tavern/generator/json/generate_json.py index 7494d0202e..154b39d859 100644 --- a/test/tavern/generator/json/generate_json.py +++ b/test/tavern/generator/json/generate_json.py @@ -359,7 +359,11 @@ def generate_semi_pair(keys, pairs, properties, number): dict: The semi-pair of values. """ result = {} + if not pairs: + raise ValueError("The 'pairs' list is empty.") for i in range(len(keys)): + if i >= len(pairs[number % len(pairs)]): + raise IndexError(f"Index {i} is out of range for 'pairs' element with length {len(pairs[number % len(pairs)])}.") result[keys[i]] = pairs[number % len(pairs)][i] if not result[keys[i]] and mode > 1: result[keys[i]] = pairs[number % len(pairs) + 1][i] diff --git a/test/tavern/generator/json/generate_prepare_index.py b/test/tavern/generator/json/generate_prepare_index.py new file mode 100644 index 0000000000..62a17b649f --- /dev/null +++ b/test/tavern/generator/json/generate_prepare_index.py @@ -0,0 +1,161 @@ +import argparse + +class Index: + def __init__(self): + """Initialize an Index instance with default values.""" + self.created = "2026-01-01 00:00:00" + self.updated = "2026-01-01 00:00:00" + self.id = 0 + self.parent = 0 + self.position = 0 + self.index_name = "" + self.index_name_english = "" + self.index_link_name = "" + self.index_link_name_english = "" + self.harvest_spec = "" + self.index_link_enabled = False + self.comment = "" + self.more_check = False + self.display_no = 5 + self.harvest_public_state = False + self.display_format = "1" + self.image_name = "" + self.public_state = False + self.public_date = None + self.recursive_public_state = False + self.rss_status = False + self.coverpage_state = False + self.recursive_coverpage_check = False + self.browsing_role = "3,-98,-99" + self.recursive_browsing_role = False + self.contribute_role = "3,4,-98" + self.recursive_contribute_role = False + self.browsing_group = "" + self.recursive_browsing_group = False + self.contribute_group = "" + self.recursive_contribute_group = False + self.owner_user_id = 1 + self.item_custom_sort = {} + self.biblio_flag = False + self.online_issn = "" + self.cnri = None + self.index_url = None + self.is_deleted = False + + def set(self, **kwargs): + """Set attributes of the Index instance. + + Args: + **kwargs: Key-value pairs of attributes to set. + """ + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + + def create_query(self): + """Generate an SQL INSERT query for the Index instance. + + Returns: + str: SQL INSERT query string. + """ + columns = [str(attr) for attr in self.__dict__.keys()] + values = [] + for col in columns: + val = getattr(self, col) + if isinstance(val, str): + values.append(f"'{val.replace("'", "''")}'") + elif isinstance(val, bool): + values.append("'true'" if val else "'false'") + elif val is None: + values.append("NULL") + elif isinstance(val, dict): + values.append(f"'{str(val).replace("'", "''")}'") + else: + values.append(str(val)) + columns_str = ", ".join(columns) + values_str = ", ".join(values) + return f"INSERT INTO index ({columns_str}) VALUES ({values_str});" + + +def create_index( + depth, num, current_level = 1, parent_id = 0, before_id = 0, adjust_position = -1 +): + """Recursively create a list of Index instances. + + Args: + depth(int): Depth of the index hierarchy. + num(int): Number of indices per level. + current_level(int, optional): Current level in the hierarchy. Defaults to 1. + parent_id(int, optional): Parent index ID. Defaults to 0. + before_id(int, optional): ID of the last created index. Defaults to 0. + adjust_position(int, optional): Adjustment for the position value. Defaults to -1. + + Returns: + list[Index]: List of created Index instances. + """ + indices = [] + id = before_id + 1 + if current_level > depth: + return indices + for i in range(1, num + 1): + index = Index() + params = { + "id": id, + "parent": parent_id, + "position": i + adjust_position, + "index_name": f"Index_L{current_level}_N{i}", + "index_name_english": f"Index_L{current_level}_N{i}_EN", + "index_link_name": f"Index_L{current_level}_N{i}_Link", + "index_link_name_english": f"Index_L{current_level}_N{i}_Link_EN", + } + index.set(**params) + indices.append(index) + child_indices = create_index(depth, num, current_level + 1, id, id) + indices.extend(child_indices) + id += len(child_indices) + 1 + return indices + + +def main(hierarchy_depth, num_indices_per_level): + """Generate SQL queries to prepare index data and save to a file. + + Args: + hierarchy_depth(int): Depth of the index hierarchy. + num_indices_per_level(int): Number of indices per level. + """ + indices = create_index(hierarchy_depth, num_indices_per_level, before_id=100, adjust_position=2) + + with open("prepare_data/prepare_index.sql", "w", encoding="utf-8") as f: + query = [index.create_query() for index in indices] + f.write("\n".join(query)) + + +def parse_args(): + """Parse command-line arguments. + + Returns: + argparse.Namespace: Parsed arguments. + """ + parser = argparse.ArgumentParser(description="Generate SQL to prepare index data.") + parser.add_argument( + "-d", + "--depth", + type=int, + default=2, + help="Hierarchy depth of indices (default: 2)", + ) + parser.add_argument( + "-n", + "--num-per-level", + type=int, + default=3, + help="Number of indices per level (default: 3)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + hierarchy_depth = args.depth + num_indices_per_level = args.num_per_level + main(hierarchy_depth, num_indices_per_level) diff --git a/test/tavern/generator/json/generate_request_body.py b/test/tavern/generator/json/generate_request_body.py new file mode 100644 index 0000000000..1972a5fee4 --- /dev/null +++ b/test/tavern/generator/json/generate_request_body.py @@ -0,0 +1,700 @@ +import argparse +import copy +import json +import os +import random +import sys +import time +from datetime import datetime +from contextlib import closing + +from psycopg2.extras import DictCursor + +sys.path.append(os.path.abspath(os.getcwd())) + +from helper.common.connect_helper import connect_db + +VALUE_ANNOTATION = "@value" +ATTRIBUTES_ANNOTATION = "@attributes" +MAPPING_DIR_BASE = "request_params/item_type_mapping" + +def remove_xsd_prefix(jpcoar_lists): + """Remove prefixes from jpcoar mapping type schemas. + + Args: + jpcoar_lists(dict): The original jpcoar mapping type schemas with prefixes. + + Returns: + dict: A new dictionary with prefixes removed from the keys. + """ + def remove_prefix(jpcoar_src, jpcoar_dst): + """Recursively remove prefixes from the keys in the jpcoar mapping type schemas. + + Args: + jpcoar_src(dict): The source dictionary with prefixes. + jpcoar_dst(dict): The destination dictionary to store the results without prefixes. + """ + for key, value in jpcoar_src.items(): + if key == 'type': + jpcoar_dst[key] = value + continue + new_key = key.split(':').pop() + jpcoar_dst[new_key] = {} + if isinstance(value, dict): + remove_prefix(value, jpcoar_dst[new_key]) + + jpcoar_copy = {} + remove_prefix(jpcoar_lists, jpcoar_copy) + return jpcoar_copy + + +def get_db_json(table, columns, where=None): + """Fetch JSON data from the database. + + Args: + table(str): The name of the database table to query. + columns(str or list): The column name(s) to retrieve. + Can be a single column name as a string or a list of column names. + where(str, optional): An optional WHERE clause to filter the query results. + Defaults to None. + + Returns: + dict or list: Fetched JSON data. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=DictCursor)) as cur: + if isinstance(columns, list): + col_str = ", ".join(columns) + else: + col_str = columns + query = f"SELECT {col_str} FROM {table}" + if where: + query += f" WHERE {where}" + cur.execute(query) + rows = cur.fetchall() + + if not rows: + return None + if len(rows) == 1: + row = rows[0] + if isinstance(columns, str) and ',' not in columns and not isinstance(columns, list): + data = row[columns] + if isinstance(data, str): + try: + data = json.loads(data) + except Exception: + pass + return data + return dict(row) + else: + return [dict(r) for r in rows] + + +def get_property_keys(schema): + """Get property keys from item schema. + + Args: + schema(dict): Item schema. + + Returns: + dict: Property keys with list of keys for each property. + """ + def collect_keys(prop, prefix=""): + """Recursively collect keys from schema property. + + Args: + prop(dict): Schema property to collect keys from. + prefix(str, optional): Prefix to add to the keys for nested properties. Defaults to "". + + Returns: + list: A list of keys collected from the schema property, with prefixes for nested properties. + """ + keys = [] + if prop.get("type") == "object" and "properties" in prop: + for k, v in prop["properties"].items(): + keys.extend(collect_keys(v, f"{prefix}{k}." if prefix else f"{k}.")) + elif prop.get("type") == "array" and "items" in prop: + items = prop["items"] + if items.get("type") in ("object", "array"): + keys.extend(collect_keys(items, prefix)) + else: + keys.append(prefix[:-1] if prefix.endswith('.') else prefix) + else: + keys.append(prefix[:-1] if prefix.endswith('.') else prefix) + return keys + + result = {} + properties = schema.get("properties", {}) + for prop_name, prop_value in properties.items(): + if prop_name == "pubdate": + result[prop_name] = [prop_name] + else: + result[prop_name] = collect_keys(prop_value) + return result + + +def find_attribute_leaves(node, path=()): + """Find leaf nodes that have attributes anywhere in the schema. + + Args: + node(dict): Current node in schema. + path(tuple, optional): Current path of keys. Defaults to (). + + Returns: + list: List of tuples (leaf_path, attribute_names). + """ + def get_attr_names(n): + """Get attribute names for a given node. + + Args: + n(dict): Node to get attribute names from. + + Returns: + list: List of attribute names for the node. + """ + if not isinstance(n, dict): + return [] + names = [] + attrs = n.get("attributes", []) + names += [attr.get("name") for attr in attrs + if isinstance(attr, dict) and "name" in attr] + if "type" in n and isinstance(n["type"], dict): + names += get_attr_names(n["type"]) + return names + + leaves = [] + if not isinstance(node, dict): + return [] + keys = [k for k in node.keys() if k != "type"] + if not keys: + attr_names = get_attr_names(node) + if attr_names: + leaves.append((path, attr_names)) + return leaves + for k in keys: + val = node[k] + if isinstance(val, dict): + leaves.extend(find_attribute_leaves(val, path + (k,))) + return leaves + + +def add_empty_mappings(mapping_item): + """Add empty string mappings for fixed mapping types. + + Args: + mapping_item(dict): Mapping item to add empty mappings to. + """ + mapping_item["display_lang_type"] = "" + mapping_item["junii2_mapping"] = "" + mapping_item["lido_mapping"] = "" + mapping_item["spase_mapping"] = "" + + +def save_body(body, output_dir, output_file_name): + """Save body to JSON file. + + Args: + body(dict): Body to save. + output_dir(str): Output directory. + output_file_name(str): Output file name. + """ + if not os.path.exists(output_dir): + os.makedirs(output_dir) + base_name, ext = os.path.splitext(output_file_name) + if not base_name.endswith("_1"): + base_name += "_1" + output_file_name = base_name + ext + output_path = os.path.join(output_dir, output_file_name) + idx = 2 + while os.path.exists(output_path): + output_path = os.path.join(output_dir, f"{base_name[:-2]}_{idx}{ext}") + idx += 1 + with open(output_path, "w", encoding="utf-8") as f: + json.dump(body, f, ensure_ascii=False, indent=2) + + +def set_nested_mapping(mapping_dict, leaf_node, value, attr_dict=None): + """Set value and attributes in nested mapping dict. + + Args: + mapping_dict(dict): Mapping dictionary to set values in. + leaf_node(str): Dot-separated path to the leaf node. + value(str): Value to set at the leaf node. + attr_dict(dict, optional): Optional attributes to set at the leaf node. + """ + keys = leaf_node.split('.') + current_dict = mapping_dict + for k in keys[:-1]: + current_dict = current_dict.setdefault(k, {}) + current_dict[keys[-1]] = {VALUE_ANNOTATION: value} + if attr_dict: + current_dict[keys[-1]][ATTRIBUTES_ANNOTATION] = attr_dict + + +def build_schema_list(schema_mt): + """Build schema list from mapping type schema. + + Args: + schema_mt(dict): Mapping type schema. + + Returns: + list: List of tuples (leaf_path, attribute_names). + """ + leaves = find_attribute_leaves(schema_mt) + result = [] + for leaf, attr_names in leaves: + result.append((".".join(leaf), attr_names)) + return result + + +def build_schema_leaf_attr_list(schema_mt): + """Build schema leaf attribute list from mapping type schema. + + Args: + schema_mt(dict): Mapping type schema. + + Returns: + list: List of tuples (leaf_path, attribute_name). + """ + leaves = find_attribute_leaves(schema_mt) + result = [] + for leaf, attr_names in leaves: + leaf_path = ".".join(leaf) + result.append((leaf_path, VALUE_ANNOTATION)) + for attr in attr_names: + result.append((leaf_path, attr)) + return result + + +def create_all_schema_mappings( + required_types, + mapping_type_schemas, + db_item_keys, + property_keys, + item_type_id, + output_dir + ): + """Create mapping body test files covering all schema leaves. + + Args: + required_types(list): List of mapping types + mapping_type_schemas(dict): Mapping type schemas + db_item_keys(list): Item keys from database + property_keys(dict): Property keys + item_type_id(int): Item type ID + output_dir(str): Output directory + """ + for mapping_type in required_types: + schema_mt = mapping_type_schemas.get(mapping_type) + if not schema_mt: + continue + schema_leaf_attr_list = build_schema_leaf_attr_list(schema_mt) + schema_leaf_attr_copy = schema_leaf_attr_list.copy() + file_idx = 1 + + while schema_leaf_attr_copy: + mapping = {} + for item in db_item_keys: + value_keys = property_keys.get(item, []) + mapping_dict = {} + v_idx = 0 + schema_leaf_attr_inner = schema_leaf_attr_copy.copy() + for leaf_path, attr in schema_leaf_attr_inner: + if v_idx >= len(value_keys): + break + keys = leaf_path.split('.') + current_dict = mapping_dict + for k in keys[:-1]: + current_dict = current_dict.setdefault(k, {}) + if attr == VALUE_ANNOTATION: + current_dict[keys[-1]] = current_dict.get(keys[-1], {}) + current_dict[keys[-1]][VALUE_ANNOTATION] = value_keys[v_idx] + else: + current_dict[keys[-1]] = current_dict.get(keys[-1], {}) + current_dict[keys[-1]].setdefault(ATTRIBUTES_ANNOTATION, {}) + current_dict[keys[-1]][ATTRIBUTES_ANNOTATION][attr] = value_keys[v_idx] + v_idx += 1 + schema_leaf_attr_copy.pop(0) + if not schema_leaf_attr_copy: + break + mapping[item] = {mt: "" for mt in required_types} + mapping[item][mapping_type] = mapping_dict if mapping_dict else "" + add_empty_mappings(mapping[item]) + + body = { + "item_type_id": item_type_id, + "mapping": mapping, + "mapping_type": mapping_type + } + save_body(body, output_dir, f"mapping_all_schemalist_{mapping_type}.json") + + +def create_duplicate_metadata(current_dict, mapping_type): + """Function to generate a mapping that causes a duplication error. + + Args: + current_dict(dict): Current mapping dictionary. + mapping_type(str): Mapping type. + + Returns: + dict: New mapping dictionary with duplication. + """ + def get_leaf_keys(d, parent_path=""): + """Recursively get leaf keys from a nested dictionary. + + Args: + d(dict): The dictionary to search for leaf keys. + parent_path(str, optional): The path to the current dictionary. Defaults to "". + + Returns: + list: A list of leaf keys in the format "key1.key2.key3". + """ + leaf_keys = [] + for key, value in d.items(): + current_path = f"{parent_path}.{key}" if parent_path else key + if isinstance(value, dict): + leaf_keys.extend(get_leaf_keys(value, current_path)) + else: + leaf_keys.append(current_path) + return leaf_keys + + def delete_key_by_path(d, path): + """Delete a key from a nested dictionary by its path. + + Args: + d(dict): The dictionary to delete the key from. + path(str): The path to the key in the format "key1.key2.key3". + """ + keys = path.split('.') + for key in keys[:-1]: + d = d.get(key, {}) + del d[keys[-1]] + + if not isinstance(current_dict, dict): + return current_dict + new_dict = copy.deepcopy(current_dict) + target_dict = new_dict[mapping_type] + if isinstance(target_dict, dict): + leaf_keys = get_leaf_keys(target_dict) + delete_key_by_path(target_dict, random.choice(leaf_keys)) + new_dict[mapping_type] = target_dict + return new_dict + +def create_mapping_body( + required_types, + mapping_type_schemas, + db_item_keys, + property_keys, + item_type_id, + output_dir, + mode="random", + same_item_keys=[] + ): + """Create mapping body test files. + + Args: + required_types(list): list of mapping types + mapping_type_schemas(dict): mapping type schemas + db_item_keys(list): item keys from database + property_keys(dict): property keys + item_type_id(int): item type ID + output_dir(str): output directory + mode(str, optional): "random" or "duplicate". Defaults to "random". + same_item_keys(list, optional): list of item keys to duplicate in "duplicate" mode. Defaults to []. + """ + file_name = "success" + if mode == 'duplicate': + file_name = mode + + # for each mapping_type, output a file + for output_mapping_type in required_types: + duplicated_key = None + mapping = {} + # Prepare a set of used schemas for each mapping_type (to prevent unexpected duplicates) + used_leaves_global = {mt: set() for mt in required_types} + for item in db_item_keys: + value_keys = property_keys.get(item, []) + if item in same_item_keys and mode == 'duplicate': + if duplicated_key is None: + duplicated_key = item + else: + duplicate_copy = copy.deepcopy(mapping[duplicated_key]) + mapping[item] = create_duplicate_metadata(duplicate_copy, output_mapping_type) + if mapping[item] is None: + mapping[item] = "" + continue + mapping[item] = {} + for mapping_type in required_types: + schema_mt = mapping_type_schemas.get(mapping_type) + if not schema_mt: + mapping[item][mapping_type] = "" + continue + schema_list = build_schema_list(schema_mt) + schema_list_copy = schema_list.copy() + random.shuffle(schema_list_copy) + mapping_dict = {} + v_idx = 0 + # Track used schema leaves to avoid duplicates per mapping_type + used_leaves = used_leaves_global[mapping_type] + used_leaves = used_leaves_global[mapping_type] + for _ in range(len(value_keys)): + available_leaves = [s for s in schema_list_copy if s[0] not in used_leaves] + if not available_leaves: + break + leaf_node, attr_names = random.choice(available_leaves) + used_leaves.add(leaf_node) + if attr_names and (len(value_keys) - v_idx) >= len(attr_names) + 1: + attr_dict = {} + for i, attr in enumerate(attr_names): + attr_dict[attr] = value_keys[v_idx + i + 1] + set_nested_mapping(mapping_dict, leaf_node, value_keys[v_idx], attr_dict) + v_idx += len(attr_names) + 1 + else: + set_nested_mapping(mapping_dict, leaf_node, value_keys[v_idx]) + v_idx += 1 + if v_idx >= len(value_keys): + break + mapping[item][mapping_type] = mapping_dict if mapping_dict else "" + add_empty_mappings(mapping[item]) + + body = { + "item_type_id": item_type_id, + "mapping": mapping, + "mapping_type": output_mapping_type + } + save_body(body, output_dir, f"mapping_{file_name}_{output_mapping_type}.json") + + +def create_mapping_body_continue(required_types, mapping_type_schemas, continue_file): + """Create mapping body test files that continue from a given file. + + Args: + required_types(list): list of mapping types + mapping_type_schemas(dict): mapping type schemas + continue_file(str): path to the continue file + + Returns: + dict: Updated mapping type schemas with used attributes removed based on the continue file. + """ + def check_and_remove_used_attributes(schema, data, parent_key=None): + """Recursively check the schema against the data and remove used attributes from the schema. + + Args: + schema(dict): The schema to check and remove attributes from. + data(dict): The data to check against the schema. + parent_key(str, optional): The parent key path for nested schemas. Defaults to None. + + Returns: + dict: The updated schema with used attributes removed. + """ + edited_schema = copy.deepcopy(schema) + for key, value in schema.items(): + if key == "type": + continue + if parent_key: + full_key = f"{parent_key}.{key}" + else: + full_key = key + if full_key not in data: + edited_schema[key] = check_and_remove_used_attributes(value, data, full_key) + else: + attribute = value["type"]["attributes"] + edited_attribute = [] + for attr in attribute: + if attr["name"] in data[full_key][ATTRIBUTES_ANNOTATION]: + edited_attribute.append(attr) + edited_schema[key]["type"]["attributes"] = edited_attribute + return edited_schema + + with open(os.path.join(MAPPING_DIR_BASE, "save_data", continue_file), "r", encoding="utf-8") as f: + continue_data = json.load(f) + for mapping_type in required_types: + mapping_type_schemas[mapping_type] = check_and_remove_used_attributes( + mapping_type_schemas[mapping_type], continue_data[mapping_type]) + return mapping_type_schemas + + +def create_error_files(required_types, item_type_id, output_dir): + """Create error test files. + + Args: + required_types(list): list of mapping types + item_type_id(int): item type ID + output_dir(str): output directory + """ + # item_type_id is invalid (string, null) + for special_id in ["abc", "null"]: + for mapping_type in required_types: + body = { + "item_type_id": None if special_id == "null" else special_id, + "mapping": {}, + "mapping_type": mapping_type + } + save_body(body, output_dir, f"mapping_{special_id}_{mapping_type}.json") + + # missing id key + for mapping_type in required_types: + body_noid = { + "mapping": {}, + "mapping_type": mapping_type + } + save_body(body_noid, output_dir, f"mapping_noid_{mapping_type}.json") + + duplicate_output_dir = output_dir.replace("error", "duplicate") + mapping = {} + if os.path.exists(duplicate_output_dir): + duplicate_files = [f for f in os.listdir(duplicate_output_dir) if f.endswith(".json")] + if duplicate_files: + with open(os.path.join(duplicate_output_dir, duplicate_files[0]), "r", encoding="utf-8") as f: + duplicate_data = json.load(f) + mapping = duplicate_data.get("mapping", {}) + # missing mapping_type key + body_noid = { + "item_type_id": item_type_id, + "mapping": mapping + } + save_body(body_noid, output_dir, f"mapping_no_mapping_type.json") + + +def parse_args(): + """Parse command line arguments. + + Returns: + argparse.Namespace: Parsed arguments. + """ + parser = argparse.ArgumentParser(description="Generate request body test files.") + parser.add_argument("item_type_id", type=int, help="target item_type_id") + parser.add_argument( + "-m", "--meta_ids", nargs=2, help="metadata IDs to duplicate", default=[]) + parser.add_argument( + "-l", "--loop_count", type=int, help="number of files to generate", default=1) + parser.add_argument( + "-c", "--continue", dest="continue_flag", action="store_true", + help="generate using unused data from previous run", default=False) + return parser.parse_args() + +def main(): + """ + Main function to generate request body test files. + Usage: python generate_request_body.py [meta_id1 meta_id2] + """ + args = parse_args() + item_type_id = args.item_type_id + meta_ids = args.meta_ids + loop_count = args.loop_count + is_continue = args.continue_flag + print(f"item_type_id: {item_type_id}, meta_ids: {meta_ids}, " + f"loop_count: {loop_count}, continue: {is_continue}") + + start_time = time.time() + print(f"テストデータ生成開始: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + total_files_before = count_json_files(os.path.join(MAPPING_DIR_BASE, "test_data")) + + required_types = ["ddi_mapping", "lom_mapping", "jpcoar_v1_mapping", "jpcoar_mapping", "oai_dc_mapping"] + test_roles = ["sysadmin", "repoadmin", "comadmin", "contributor", "user", "guest"] + + exists = get_db_json("item_type", "id", f"id = {item_type_id}") + if not exists: + print(f"Error: item_type_id {item_type_id} does not exist in item_type table.") + sys.exit(1) + + schemas_raw = get_db_json("oaiserver_schema", ["xsd", "schema_name"]) + mapping_type_schemas = {} + if schemas_raw: + for row in schemas_raw: + mapping_type = row.get("schema_name") + xsd = row.get("xsd") + if isinstance(xsd, str): + xsd = json.loads(xsd) + mapping_type_schemas[mapping_type] = remove_xsd_prefix(xsd) + + schema = get_db_json("item_type", "schema", f"id = {item_type_id}") + property_keys = get_property_keys(schema) + + # Metadata ID List + mapping_raw = get_db_json("item_type_mapping", "mapping", f"item_type_id = {item_type_id}") + if not mapping_raw: + raise Exception(f"mapping not found for item_type_id={item_type_id}") + while isinstance(mapping_raw, list): + if not mapping_raw: + raise Exception(f"mapping is empty for item_type_id={item_type_id}") + mapping_raw = mapping_raw[0] + if isinstance(mapping_raw, dict) and "mapping" in mapping_raw: + mapping_raw = mapping_raw["mapping"] + + # Organize Metadata ID List + def set_db_keys(k): + if k == "pubdate": + return (0, "") + elif k == "system_file": + return (1, k) + elif k.startswith("system_identifier"): + return (2, k) + elif k.startswith("item_"): + # Sort by numerical part in ascending order + try: + num = int(k.split("_")[1]) + except Exception: + num = 0 + return (3, num) + else: + return (4, k) + db_item_keys = sorted(list(mapping_raw.keys()), key=set_db_keys) + if meta_ids and len(meta_ids) == 2: + db_item_keys = [id for id in meta_ids if id in db_item_keys] + [k for k in db_item_keys if k not in meta_ids] + + for role in test_roles: + print(f"{role}のテストデータの生成を開始します。") + # Generate valid data using all schema lists + output_dir = os.path.join(MAPPING_DIR_BASE, "test_data", role, "all_schema_mappings") + create_all_schema_mappings(required_types, mapping_type_schemas, db_item_keys, property_keys, item_type_id, output_dir) + + # Generate JSON data that will succeed (values are random and do not overlap with others) + output_dir = os.path.join(MAPPING_DIR_BASE, "test_data", role, "success") + if is_continue: + save_data_file = sorted([f for f in os.listdir(os.path.join(MAPPING_DIR_BASE, "save_data")) + if f.startswith(f"unused_schema_{item_type_id}_{role}_") and f.endswith(".json")], reverse=True) + mapping_type_schemas = create_mapping_body_continue(required_types, mapping_type_schemas, save_data_file[0]) + for _ in range(loop_count): + create_mapping_body( + required_types, mapping_type_schemas, db_item_keys, property_keys, + item_type_id, output_dir, mode="random" + ) + + # Generate JSON data with duplicate metadata IDs (values other than specified metadata are random and do not overlap) + if meta_ids and len(meta_ids) == 2: + output_dir = os.path.join(MAPPING_DIR_BASE, "test_data", role, "duplicate") + same_item_keys = sorted(meta_ids, reverse=False) + create_mapping_body(required_types, mapping_type_schemas, db_item_keys, + property_keys, item_type_id, output_dir, mode="duplicate", + same_item_keys=same_item_keys) + else: + print("メタデータIDの指定が不正です。重複エラー用のテストデータは生成しません。") + + # Generate test files for error cases + output_dir = os.path.join(MAPPING_DIR_BASE, "test_data", role, "error") + create_error_files(required_types, item_type_id, output_dir) + print(f"{role}のテストデータの生成が終了しました。") + + total_files_after = count_json_files(os.path.join(MAPPING_DIR_BASE, "test_data")) + elapsed = time.time() - start_time + print(f"テストデータ生成終了: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"所要時間: {int(elapsed // 60)}分 {int(elapsed % 60)}秒") + print(f"生成したJSONファイル数: {total_files_after - total_files_before}件") + +def count_json_files(root_dir): + """Count the number of JSON files in a directory and its subdirectories. + + Args: + root_dir(str): The root directory to start counting from. + + Returns: + int: The total count of JSON files found. + """ + count = 0 + for _, _, filenames in os.walk(root_dir): + count += sum(1 for f in filenames if f.endswith('.json')) + return count + +if __name__ == "__main__": + main() diff --git a/test/tavern/generator/json/template_file/index_template.json b/test/tavern/generator/json/template_file/index_template.json new file mode 100644 index 0000000000..b82ee952d2 --- /dev/null +++ b/test/tavern/generator/json/template_file/index_template.json @@ -0,0 +1,26 @@ +{ + "browsing_group": "Group", + "browsing_role": "Role", + "can_edit": "bool", + "cnri": "str", + "comment": "str", + "contribute_group": "Group", + "contribute_role": "Role", + "coverpage_state": "bool", + "display_format": "str", + "display_no": "int", + "harvest_public_state": "bool", + "harvest_spec": "str", + "have_children": "bool", + "index_link_enabled": "bool", + "index_link_name": "str", + "index_link_name_english": "str", + "index_name": "str", + "index_name_english": "str", + "index_url": "str", + "more_check": "bool", + "online_issn": "str", + "public_date": "date", + "public_state": "bool", + "rss_status": "bool" +} diff --git a/test/tavern/helper/common/connect_helper.py b/test/tavern/helper/common/connect_helper.py new file mode 100644 index 0000000000..62cd500478 --- /dev/null +++ b/test/tavern/helper/common/connect_helper.py @@ -0,0 +1,34 @@ +import psycopg2 +import redis + +from helper.config import DATABASE, REDIS + + +def connect_db(): + """Connect to the PostgreSQL database. + + Returns: + psycopg2.extensions.connection: Connection to the database. + """ + conn = psycopg2.connect( + dbname=DATABASE["dbname"], + user=DATABASE["user"], + password=DATABASE["password"], + host=DATABASE["host"], + port=DATABASE["port"], + ) + return conn + + +def connect_redis(db=0): + """Connect to the Redis server. + + Returns: + redis.Redis: Connection to the Redis server. + """ + r = redis.Redis( + host=REDIS["host"], + port=REDIS["port"], + db=db, + ) + return r diff --git a/test/tavern/helper/common/item_create_helper.py b/test/tavern/helper/common/item_create_helper.py new file mode 100644 index 0000000000..77eace7023 --- /dev/null +++ b/test/tavern/helper/common/item_create_helper.py @@ -0,0 +1,571 @@ +import json +import requests +from os import path + +from helper.common.request_helper import ( + request_create_action_param, + request_create_deposits_items_index_param, + request_create_deposits_items_param, + request_create_deposits_redirect_param, + request_create_save_activity_data_param, + request_create_save_param, + request_create_validate_param, +) +from helper.common.response_helper import ( + response_save_file_upload_info, + response_save_identifier_grant, + response_save_next_path, + response_save_recid, + response_save_tree_data, + response_save_url, +) + + +def create_item( + response, + host, + create_info_file, + creation_count, + target_index=None, + file_path=None, + is_doi=False, + prepare_edit=False, + id=2000001, + replace_title=True, +): + """Create items based on the provided creation information. + + Args: + response(Response): The response object from the initial request. + host(str): The base URL of the WEKO instance. + create_info_file(str): Path to the JSON file containing creation information. + creation_count(int): Number of items to create. + target_index(int, optional): The target index ID to assign to the created items. If None, no index assignment is made. + file_path(str, optional): Path to the file to upload with the item. + is_doi(bool, optional): Boolean indicating whether to assign a DOI to the item. + prepare_edit(bool, optional): Boolean indicating whether to prepare the item for editing after creation. + id(int, optional): Optional ID to assign to the created item. + replace_title(bool, optional): Boolean indicating whether to replace the title of the item. + + Raises: + Exception: If any step in the item creation process fails. + """ + # Get the necessary headers from the response + request_headers = response.request.headers + header = { + "Cookie": request_headers.get("Cookie", ""), + "X-CSRFToken": request_headers.get("X-CSRFToken", ""), + } + + with open(create_info_file, "r") as f: + create_info = json.loads(f.read()) + + session = requests.Session() + session.headers.update(header) + with open(create_info["data_file"], "r") as f: + data = json.loads(f.read()) + + if create_info.get("file_key") and data.get(create_info["file_key"]) is not None: + url = data[create_info["file_key"]][0]["url"]["url"] + replaced_url = url.replace("{id}", str(id)) + data[create_info["file_key"]][0]["url"]["url"] = replaced_url + + if replace_title: + title_key = create_info["title_key"].split(".") + title = data[title_key[0]][int(title_key[1])][title_key[2]] + replaced_title = title + f"_{id}" + data[title_key[0]][int(title_key[1])][title_key[2]] = replaced_title + + if create_info.get("identifier_key") and data.get(create_info["identifier_key"]) is not None: + identifier = data[create_info["identifier_key"]][0]["subitem_identifier_uri"] + replaced_identifier = identifier + f"_{id}" + data[create_info["identifier_key"]][0]["subitem_identifier_uri"] = replaced_identifier + + for _ in range(creation_count): + # create activity + activity_response = activity_init( + host, + session, + create_info["flow_id"], + create_info["itemtype_id"], + create_info["workflow_id"], + ) + + # next path + next_path(host, session, activity_response["next_path"]) + + if file_path: + # deposits item + deposits_response = deposits_items(host, session) + + # upload file to bucket + file_upload_info = bucket_file( + deposits_response["url"]["bucket"], session, file_path + ) + + # item validate + item_validate( + host, + session, + data, + file_metadata=file_upload_info["file_metadata"], + ) + + # save activity data + save_activity_data( + host, + session, + activity_response["activity_id"], + data, + create_info["title_key"], + ) + + # iframe model save + iframe_model_save( + host, + session, + data, + url=json.dumps(deposits_response["url"]), + file_upload_info=file_upload_info["file_upload_info"], + file_metadata=file_upload_info["file_metadata"], + ) + + else: + # item_validate + item_validate(host, session, data) + + # save activity data + save_activity_data( + host, + session, + activity_response["activity_id"], + data, + create_info["title_key"], + ) + + # iframe model save + iframe_model_save(host, session, data) + + # deposits item + deposits_response = deposits_items(host, session, data) + + # deposits redirect + deposits_redirect( + host, + session, + deposits_response["recid"], + data, + create_info["title_key"], + ) + + no_random = False + if target_index: + no_random = True + if isinstance(target_index, int): + tree_response = {"tree_data": [str(target_index)]} + elif isinstance(target_index, list): + tree_response = {"tree_data": [str(i) for i in target_index]} + else: + # api tree + tree_response = api_tree(host, session, deposits_response["recid"]) + # deposits items recid + deposits_items_recid( + host, + session, + deposits_response["recid"], + str(tree_response["tree_data"]), + no_random, + ) + + # activity 3 + activity_3( + host, + session, + activity_response["activity_id"], + create_info["action_version"]["3"], + ) + + # activity 5 + activity_5( + host, + session, + activity_response["activity_id"], + create_info["action_version"]["5"], + ) + + # activity detail + activity_detail_response = activity_detail( + host, session, activity_response["activity_id"] + ) + + # activity 7 + activity_7( + host, + session, + activity_response["activity_id"], + create_info["action_version"]["7"], + activity_detail_response["identifier_grant"], + is_doi=is_doi, + ) + + # activity 4 + activity_4( + host, + session, + activity_response["activity_id"], + create_info["action_version"]["4"], + ) + + if prepare_edit: + # prepare edit item + prepare_edit_item(host, session, deposits_response["recid"]) + + +def activity_init(host, session, flow_id, itemtype_id, workflow_id): + """Initialize a workflow activity. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + flow_id(str): The ID of the workflow flow. + itemtype_id(str): The ID of the item type. + workflow_id(str): The ID of the workflow. + + Returns: + dict: The response containing the activity ID and next path. + + Raises: + Exception: If the request fails or the response is not as expected. + """ + url = f"{host}/workflow/activity/init" + data = { + "flow_id": flow_id, + "itemtype_id": itemtype_id, + "workflow_id": workflow_id, + } + response = session.post(url, json=data, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to initialize activity: {response.text}") + return response_save_next_path(response) + + +def next_path(host, session, path): + """Get the next path in the workflow. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + path(str): The next path to navigate to. + + Raises: + Exception: If the request fails or the response is not as expected. + """ + url = f"{host}{path}" + response = session.get(url, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to get next path: {response.text}") + + +def item_validate(host, session, data, file_metadata=None): + """Validate item data. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + data(dict): The item data written in json string. + file_metadata(str, optional): File upload info written in json string. + + Raises: + Exception: If the validation fails or the response is not as expected. + """ + params = request_create_validate_param(data, file_metadata) + url = f"{host}/api/items/validate" + response = session.post(url, json=params, verify=False) + + if response.status_code != 200: + raise Exception(f"Validation failed: {response.text}") + + +def save_activity_data(host, session, activity_id, data, title_key): + """Save activity data. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to save data for. + data(dict): The activity data written in json string. + title_key(str): The key for the title in the data. + + Raises: + Exception: If the save operation fails or the response is not as expected. + """ + params = request_create_save_activity_data_param(activity_id, data, title_key) + url = f"{host}/workflow/save_activity_data" + response = session.post(url, json=params, verify=False) + + if response.status_code != 200: + raise Exception(f"Failed to save activity data: {response.text}") + + +def iframe_model_save( + host, + session, + data, + url=None, + file_upload_info=None, + file_metadata=None, +): + """Save item data in iframe model format. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + data(dict): The item data written in json string. + url(str, optional): URL written in json string. + file_upload_info(str, optional): File upload info written in json string. + file_metadata(str, optional): File metadata written in json string. + + Raises: + Exception: If the save operation fails or the response is not as expected. + """ + params = request_create_save_param(data, url, file_upload_info, file_metadata) + url = f"{host}/items/iframe/model/save" + response = session.post(url, json=params, verify=False) + + if response.status_code != 200: + raise Exception(f"Failed to save item: {response.text}") + + +def deposits_items(host, session, data=None): + """Deposit items based on the provided data file. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + data(dict, optional): The item data written in json string. + + Returns: + dict: The response containing the recid of the deposited item. + + Raises: + Exception: If the deposit operation fails or the response is not as expected. + """ + params = {} + if data is not None: + params = request_create_deposits_items_param(data) + + url = f"{host}/api/deposits/items" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to deposit item: {response.text}") + + if data: + return response_save_recid(response) + else: + return response_save_url(response) + + +def deposits_redirect(host, session, recid, data, title_key): + """Redirect a deposit based on the provided recid and data file. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + recid(str): The recid of the deposit to redirect. + data(dict): The data for redirection written in json string or dict. + title_key(str): The key for the title in the data. + + Raises: + Exception: If the redirection fails or the response is not as expected. + """ + params = request_create_deposits_redirect_param(data, title_key) + url = f"{host}/api/deposits/redirect/{recid}" + response = session.put(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to redirect deposit: {response.text}") + + +def api_tree(host, session, recid): + """Get the tree data for a specific recid. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + recid(str): The recid of the item to get tree data for. + + Returns: + dict: The response containing the tree data. + + Raises: + Exception: If the request fails or the response is not as expected. + """ + url = f"{host}/api/tree/{recid}" + response = session.get(url, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to get tree: {response.text}") + + return response_save_tree_data(response) + + +def deposits_items_recid(host, session, recid, tree_data, no_random): + """Update deposits items with the provided recid and tree data. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + recid(str): The recid of the item to update. + tree_data(str): The tree data to update the item with. + no_random(bool): no random choice if True + + Raises: + Exception: If the update operation fails or the response is not as expected. + """ + params = request_create_deposits_items_index_param(tree_data, no_random) + url = f"{host}/api/deposits/items/{recid}" + response = session.put(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to get deposits items: {response.text}") + + +def activity_3(host, session, activity_id, version): + """Perform activity action 3. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to perform action on. + version(str): The version of the action to perform. + + Raises: + Exception: If the action fails or the response is not as expected. + """ + params = request_create_action_param(version) + url = f"{host}/workflow/activity/action/{activity_id}/3" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to perform activity action: {response.text}") + + +def activity_5(host, session, activity_id, version): + """Perform activity action 5. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to perform action on. + version(str): The version of the action to perform. + + Raises: + Exception: If the action fails or the response is not as expected. + """ + params = request_create_action_param(version, link_data=[]) + url = f"{host}/workflow/activity/action/{activity_id}/5" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to perform activity action: {response.text}") + + +def activity_detail(host, session, activity_id): + """Get the details of a specific activity. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to get details for. + + Returns: + dict: The response containing the activity details, including identifier grant. + + Raises: + Exception: If the request fails or the response is not as expected. + """ + url = f"{host}/workflow/activity/detail/{activity_id}?page=1&size=20" + response = session.get(url, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to get activity detail: {response.text}") + + return response_save_identifier_grant(response) + + +def activity_7(host, session, activity_id, version, identifier, is_doi=False): + """Perform activity action 7. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to perform action on. + version(str): The version of the action to perform. + identifier(str): The identifier to use in the action. + is_doi(bool): Whether to assign a DOI. + + Raises: + Exception: If the action fails or the response is not as expected. + """ + params = request_create_action_param(version, identifier=identifier, is_doi=is_doi) + url = f"{host}/workflow/activity/action/{activity_id}/7" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to perform activity action: {response.text}") + + +def activity_4(host, session, activity_id, version): + """Perform activity action 4. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + activity_id(str): The ID of the activity to perform action on. + version(str): The version of the action to perform. + + Raises: + Exception: If the action fails or the response is not as expected. + """ + params = request_create_action_param(version, community="") + url = f"{host}/workflow/activity/action/{activity_id}/4" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to perform activity action: {response.text}") + + +def bucket_file(url, session, file_path): + """Upload a file to the specified bucket URL. + + Args: + url(str): The bucket URL to upload the file to. + session(requests.Session): The session object for making requests. + file_path(str): The path to the file to upload. + + Returns: + dict: The response containing file upload info and metadata. + Raises: + Exception: If the file upload fails. + """ + file_name = file_path.split("/")[-1] + response = session.put( + path.join(url, file_name), + headers={"Content-Type": "text/plain"}, + files={"file": (file_name, open(file_path, "rb"))}, + verify=False, + ) + if response.status_code != 200: + raise Exception(f"Failed to upload file: {response.text}") + + return response_save_file_upload_info(response, "item_30002_file35", 30002) + + +def prepare_edit_item(host, session, recid): + """Prepare an item for editing. + + Args: + host(str): The base URL of the WEKO instance. + session(requests.Session): The session object for making requests. + recid(str): The recid of the item to prepare for editing. + + Raises: + Exception: If the prepare edit operation fails. + """ + params = {"pid_value": recid} + url = f"{host}/items/prepare_edit_item" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to prepare edit item: {response.text}") diff --git a/test/tavern/helper/common/item_import_helper.py b/test/tavern/helper/common/item_import_helper.py new file mode 100644 index 0000000000..b07684c28b --- /dev/null +++ b/test/tavern/helper/common/item_import_helper.py @@ -0,0 +1,160 @@ +import csv +import os +import zipfile +from time import sleep + +import requests +from box import Box + + +def import_item(response, host, file_path, target_index, csrf_token): + """Import item via admin API. + + Args: + response(Response): Response object from previous request to get headers. + host(str): Host URL. + file_path(str): Path to the file to be imported. + target_index(int): Target index ID to replace in the import data. + csrf_token(str): CSRF token for authentication. + + Returns: + Box: A Box object containing import task details. + """ + request_headers = response.request.headers + header = { + "Referer": host, + "Cookie": request_headers.get("Cookie", ""), + "X-CSRFToken": csrf_token, + } + + session = requests.Session() + session.headers.update(header) + + # Prepare import data + import_data_path = prepare_import_data(file_path, target_index) + + # Import check + check_import_task_id = import_check(host, session, import_data_path) + + # Get check status + while True: + check_result = get_check_status(host, session, check_import_task_id) + if check_result.get("end_date"): + break + sleep(1) + + # Import + import_response = start_import( + host, session, check_result["data_path"], check_result["list_record"] + ) + + return Box({"import_tasks": import_response["data"]["tasks"]}) + + +def prepare_import_data(file_path, target_index): + """Prepare import data by replacing placeholder with target index ID + and zipping the data directory. + + Args: + file_path(str): Path to the file to be prepared. + target_index(int): Target index ID to replace in the import data. + + Returns: + str: Path to the zipped import data. + """ + replace_path = file_path.replace("prepare", "data") + with open(file_path, "r", encoding="utf-8") as read_file, \ + open(replace_path, "w", encoding="utf-8") as write_file: + reader = csv.reader(read_file, delimiter="\t") + writer = csv.writer(write_file, delimiter="\t") + + for row in reader: + new_row = [cell.replace("<インデックスID>", str(target_index)) for cell in row] + writer.writerow(new_row) + + target_dir = "/".join(replace_path.split("/")[:-1]) + output_zip_path = target_dir + ".zip" + with zipfile.ZipFile(output_zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(target_dir): + for file in files: + file_full_path = os.path.join(root, file) + arcname = os.path.relpath(file_full_path, os.path.dirname(target_dir)) + zipf.write(file_full_path, arcname) + + return output_zip_path + + +def import_check(host, session, file_path): + """Check import item via admin API. + + Args: + host(str): Host URL. + session(requests.Session): Requests session with appropriate headers. + file_path(str): Path to the file to be checked. + + Returns: + str: Check import task ID. + + Raises: + Exception: If the request fails. + """ + url = f"{host}/admin/items/import/check" + params = {"is_change_identifier": "false"} + with open(file_path, "rb") as file: + files = { + "file": (file_path.split("/")[-1], file, "application/x-zip-compressed"), + } + response = session.post(url, data=params, files=files, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to check import item: {response.text}") + + return response.json()["check_import_task_id"] + + +def get_check_status(host, session, check_import_task_id): + """Get check import status via admin API. + + Args: + host(str): Host URL. + session(requests.Session): Requests session with appropriate headers. + check_import_task_id(str): Check import task ID. + + Returns: + dict: Check import status details. + + Raises: + Exception: If the request fails. + """ + params = {"task_id": check_import_task_id} + url = f"{host}/admin/items/import/get_check_status" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to get check status: {response.text}") + return response.json() + + +def start_import(host, session, data_path, list_record): + """Start import item via admin API. + + Args: + host(str): Host URL. + session(requests.Session): Requests session with appropriate headers. + data_path(str): Data path for import. + list_record(list): List of records to be imported. + + Returns: + dict: Import task details. + + Raises: + Exception: If the request fails. + """ + params = { + "data_path": data_path, + "list_doi": ["" for _ in range(len(list_record))], + "list_record": list_record, + } + url = f"{host}/admin/items/import/import" + response = session.post(url, json=params, verify=False) + if response.status_code != 200: + raise Exception(f"Failed to start import: {response.text}") + return response.json() diff --git a/test/tavern/helper/request_helper.py b/test/tavern/helper/common/request_helper.py similarity index 70% rename from test/tavern/helper/request_helper.py rename to test/tavern/helper/common/request_helper.py index 5fcff54a99..a1c03ea2a3 100644 --- a/test/tavern/helper/request_helper.py +++ b/test/tavern/helper/common/request_helper.py @@ -4,17 +4,18 @@ def request_create_validate_param(data, file_metadata=None): """Create params for {host}/api/items/validate - + Args: - data (str): register data written in json string - file_upload_info (str): file upload info written in json string - + data(str): register data written in json string + file_upload_info(str, optional): file upload info written in json string + Returns: dict: params for {host}/api/items/validate item_id (str): item id data (dict): register data """ - data = json.loads(data) + if isinstance(data, str): + data = json.loads(data) item_id = data['$schema'].split('/')[-1] params = { 'item_id': item_id, @@ -23,24 +24,27 @@ def request_create_validate_param(data, file_metadata=None): if file_metadata is not None: metadata = json.loads(file_metadata) keys = list(metadata.keys()) - params['data'][keys[0]] = metadata[keys[0]] + if keys[0] not in params['data'].keys(): + params['data'][keys[0]] = metadata[keys[0]] return params + def request_create_save_activity_data_param(activity_id, data, title_key): """ Create params for {host}/workflow/save_activity_data - + Args: - activity_id (str): activity_id - data (str): register data written in json string - title_key (str): key of title from item type schema - + activity_id(str): activity_id + data(str|dict): register data written in json string + title_key(str): key of title from item type schema + Returns: dict: params for {host}/workflow/save_activity_data activity_id (str): activity id - shared_user_id (int): shared user id + shared_user_ids (list): shared user ids title (str): title """ - data = json.loads(data) + if isinstance(data, str): + data = json.loads(data) try: if len(title_key.split('.')) == 3: title = data[title_key.split('.')[0]][int(title_key.split('.')[1])][title_key.split('.')[2]] @@ -50,26 +54,28 @@ def request_create_save_activity_data_param(activity_id, data, title_key): title = '' return { 'activity_id': activity_id, - 'shared_user_id': data['shared_user_id'], + 'shared_user_ids': data['shared_user_ids'], 'title': title } + def request_create_save_param(data, url=None, file_upload_info=None, file_metadata=None): """Create params for {host}/items/iframe/model/save - + Args: - data (str): register data written in json string - url (str): url written in json string - file_upload_info (str): file upload info written in json string - file_metadata (str): file metadata written in json string - + data(str|dict): register data written in json string + url(str, optional): url written in json string + file_upload_info(str, optional): file upload info written in json string + file_metadata(str, optional): file metadata written in json string + Returns: dict: params for {host}/items/iframe/model/save endpoints (dict): endpoints files (list): files metainfo (dict): metainfo """ - data = json.loads(data) + if isinstance(data, str): + data = json.loads(data) params = { 'endpoints': { 'initialization': '/api/deposits/items' @@ -92,44 +98,49 @@ def request_create_save_param(data, url=None, file_upload_info=None, file_metada if file_metadata is not None: metadata = json.loads(file_metadata) keys = list(metadata.keys()) - params['metainfo'][keys[0]] = metadata[keys[0]] + if keys[0] not in params['metainfo'].keys(): + params['metainfo'][keys[0]] = metadata[keys[0]] return params def request_create_deposits_items_param(data=None): """Create params for {host}/api/deposits/items - + Args: - data (str): register data written in json string - + data(str|dict, optional): register data written in json string + Returns: dict: params for {host}/api/deposits/items $schema (str): schema """ if data is None: return {} - data = json.loads(data) + if isinstance(data, str): + data = json.loads(data) return { '$schema': data['$schema'] } + def request_create_deposits_redirect_param(data, title_key, file_metadata=None): """Create params for {host}/api/deposits/redirect/{recid} - + Args: - data (str): register data written in json string - title_key (str): key of title from item type schema - + data(str|dict): register data written in json string + title_key(str): key of title from item type schema + file_metadata(str, optional): file metadata written in json string + Returns: dict: params for {host}/api/deposits/redirect/{recid} $schema (str): schema lang (str): lang pubdate (str): publish date - shared_user_id (int): shared user id + shared_user_ids (list): shared user ids title (str): title [key] (list or dict): item data with value - deleted_items (list): item keys with no value + deleted_items (list): item keys with no value """ - data = json.loads(data) + if isinstance(data, str): + data = json.loads(data) if file_metadata is not None: metadata = json.loads(file_metadata) keys = list(metadata.keys()) @@ -147,7 +158,7 @@ def request_create_deposits_redirect_param(data, title_key, file_metadata=None): '$schema': data['$schema'], 'lang': 'ja', 'pubdate': data['pubdate'], - 'shared_user_id': data['shared_user_id'], + 'shared_user_ids': data['shared_user_ids'], 'title': title } @@ -163,12 +174,14 @@ def request_create_deposits_redirect_param(data, title_key, file_metadata=None): return_params['deleted_items'] = deleted_items return return_params -def request_create_deposits_items_index_param(indexes): + +def request_create_deposits_items_index_param(indexes, no_random): """Create params for {host}/api/deposits/items/{recid} - + Args: - indexes (str): index id list written in string - + indexes(str): index id list written in string + no_random(bool): no random choice if True + Returns: dict: params for {host}/api/deposits/items/{recid} actions (str): actions @@ -176,18 +189,26 @@ def request_create_deposits_items_index_param(indexes): """ return { 'actions': 'private', - 'index': [random.choice(eval(indexes))] + 'index': [random.choice(eval(indexes))] if not no_random else eval(indexes) } -def request_create_action_param(action_version, link_data = None, identifier = None, community = None): + +def request_create_action_param( + action_version, + link_data = None, + identifier = None, + community = None, + is_doi=False + ): """Create params for {host}/workflow/activity/action/{activity_id}/{action_id} - + Args: - action_version (str): action version - link_data (str): link data written in json string - identifier (str): identifier written in json string - community (str): community id - + action_version(str): action version + link_data(str, optional): link data written in json string + identifier(str, optional): identifier written in json string + community(str, optional): community id + is_doi(bool, optional): is doi granted + Returns: dict: params for {host}/workflow/activity/action/{activity_id}/{action_id} action_version (str): action version @@ -213,15 +234,19 @@ def request_create_action_param(action_version, link_data = None, identifier = N return_params['link_data'] = link_data if identifier is not None: identifier = json.loads(identifier) - return_params['identifier_grant'] = '0' + return_params['identifier_grant'] = '1' if is_doi else '0' return_params['identifier_grant_crni_link'] = identifier['crni_link'] - return_params['identifier_grant_jalc_cr_doi_link'] = identifier['jalc_cr_doi_link'] + identifier['jalc_cr_doi_suffix'] + return_params['identifier_grant_jalc_cr_doi_link'] = \ + identifier['jalc_cr_doi_link'] + identifier['jalc_cr_doi_suffix'] return_params['identifier_grant_jalc_cr_doi_suffix'] = identifier['jalc_cr_doi_suffix'] - return_params['identifier_grant_jalc_dc_doi_link'] = identifier['jalc_dc_doi_link'] + identifier['jalc_dc_doi_suffix'] + return_params['identifier_grant_jalc_dc_doi_link'] = \ + identifier['jalc_dc_doi_link'] + identifier['jalc_dc_doi_suffix'] return_params['identifier_grant_jalc_dc_doi_suffix'] = identifier['jalc_dc_doi_suffix'] - return_params['identifier_grant_jalc_doi_link'] = identifier['jalc_doi_link'] + identifier['jalc_doi_suffix'] + return_params['identifier_grant_jalc_doi_link'] = \ + identifier['jalc_doi_link'] + identifier['jalc_doi_suffix'] return_params['identifier_grant_jalc_doi_suffix'] = identifier['jalc_doi_suffix'] - return_params['identifier_grant_ndl_jalc_doi_link'] = identifier['ndl_jalc_doi_link'] + identifier['ndl_jalc_doi_suffix'] + return_params['identifier_grant_ndl_jalc_doi_link'] = \ + identifier['ndl_jalc_doi_link'] + identifier['ndl_jalc_doi_suffix'] return_params['identifier_grant_ndl_jalc_doi_suffix'] = identifier['ndl_jalc_doi_suffix'] if community is not None: return_params['community'] = community @@ -230,13 +255,14 @@ def request_create_action_param(action_version, link_data = None, identifier = N return_params['temporary_save'] = 0 return return_params + def request_create_author_edit_param(file_name, author_id): """Create params for {host}/api/author/edit - + Args: - file_name (str): file name of author register data - author_id (str): author id - + file_name(str): file name of author register data + author_id(str): author id + Returns: dict: params for {host}/api/author/edit author (dict): author register data diff --git a/test/tavern/helper/response_helper.py b/test/tavern/helper/common/response_helper.py similarity index 81% rename from test/tavern/helper/response_helper.py rename to test/tavern/helper/common/response_helper.py index b4e994efe0..2f5c955ac4 100644 --- a/test/tavern/helper/response_helper.py +++ b/test/tavern/helper/common/response_helper.py @@ -1,21 +1,22 @@ -from bs4 import BeautifulSoup -from box import Box from datetime import datetime, timezone import json +import os import random import string -from urllib.parse import urlencode, urlparse +from urllib.parse import urlparse + +from bs4 import BeautifulSoup +from box import Box -from helper.config import RESOURCE_TYPE_URI, SWORD_CONFIG_FILE -from helper.verify_database_helper import connect_db +from helper.config import RESOURCE_TYPE_URI def response_save_next_path(response): """Save data from "{host}/workflow/activity/init"'s response - + Args: - response (requests.models.Response): response from {host}/workflow/activity/init - + response(requests.models.Response): response from {host}/workflow/activity/init + Returns: Box: next_path and activity_id next_path (str): next path @@ -28,12 +29,13 @@ def response_save_next_path(response): 'activity_start_time': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') }) + def response_save_recid(response): """Save data from "{host}/api/deposits/items"'s response - + Args: - response (requests.models.Response): response from {host}/api/deposits/items - + response(requests.models.Response): response from {host}/api/deposits/items + Returns: Box: recid recid (str): recid @@ -41,13 +43,14 @@ def response_save_recid(response): json = response.json() return Box({'recid': json['id']}) -def response_save_register_data(response, file_name): + +def response_save_register_data(_, file_name): """Save data from target file - + Args: - response (requests.models.Response): response from {host}{next_path} - file_name (str): name of the file containing the data to be registered - + _(requests.models.Response): response from {host}{next_path} + file_name(str): name of the file containing the data to be registered + Returns: Box: register_data register_data (str): register data @@ -55,12 +58,13 @@ def response_save_register_data(response, file_name): with open('request_params/' + file_name, 'r') as f: return Box({'register_data': f.read()}) + def response_save_tree_data(response): """Save data from "{host}/api/tree/{recid}"'s response - + Args: - response (requests.models.Response): response from {host}/api/tree/{recid} - + response(requests.models.Response): response from {host}/api/tree/{recid} + Returns: Box: tree_data tree_data (list): index id list @@ -68,12 +72,13 @@ def response_save_tree_data(response): json = response.json() return Box({'tree_data': [t['id'] for t in json]}) + def response_save_identifier_grant(response): """Save data from "{host}/workflow/activity/detail/{activity_id}?page=1&size=20"'s response - + Args: - response (requests.models.Response): response from {host}/workflow/activity/detail/{activity_id}?page=1&size=20 - + response(requests.models.Response): response from {host}/workflow/activity/detail/{activity_id}?page=1&size=20 + Returns: Box: identifier_grant identifier_grant (str): identifier grant written in json string @@ -119,11 +124,12 @@ def response_save_identifier_grant(response): }) }) + def response_save_author_prefix_settings(response): """Save data from "{host}/api/items/author_prefix_settings"'s response Args: - response (requests.models.Response): response from {host}/api/items/author_prefix_settings + response(requests.models.Response): response from {host}/api/items/author_prefix_settings Returns: Box: settings @@ -133,24 +139,26 @@ def response_save_author_prefix_settings(response): settings.insert(0, None) return Box({'settings': settings}) + def response_save_group_list(response): """Save data from "{host}/accounts/settings/groups/grouplist"'s response Args: - response (requests.models.Response): response from {host}/accounts/settings/groups/grouplist - + response(requests.models.Response): response from {host}/accounts/settings/groups/grouplist + Returns: Box: group_list group_list (list): group list """ return Box({'group_list': list(response.json().keys())}) + def response_save_url(response): """Save data from "{host}/api/deposits/items"'s response Args: - response (requests.models.Response): response from {host}/api/deposits/items - + response(requests.models.Response): response from {host}/api/deposits/items + Returns: Box: url url (dict): url lists @@ -159,14 +167,15 @@ def response_save_url(response): recid = url['r'].split('/')[-1] return Box({'url': url, 'recid': recid}) + def response_save_file_upload_info(response, file_key, item_id): """Save data from "{url.bucket}"/[file_name]"'s response - + Args: - response (requests.models.Response): response from {url.bucket}/[file_name] - file_key (str): key of file - item_id (str): item id - + response(requests.models.Response): response from {url.bucket}/[file_name] + file_key(str): key of file + item_id(str): item id + Returns: Box: file_upload_info (dict): file upload info @@ -181,7 +190,7 @@ def response_save_file_upload_info(response, file_key, item_id): if key == 'url': parse = urlparse(file_upload_info['links']['self']) url = parse.scheme + '://' + parse.netloc + '/record/2000001/files/' + file_upload_info['key'] - file_metadata[key] = {'url': url} + file_metadata[key] = {'url': url} elif key == 'date': created_str = file_upload_info['created'] created = datetime.strptime(created_str, '%Y-%m-%dT%H:%M:%S.%f%z').date() @@ -205,13 +214,17 @@ def response_save_file_upload_info(response, file_key, item_id): file_metadata[key] = 'open_access' file_metadata['version_id'] = file_upload_info['version_id'] - return Box({'file_upload_info': json.dumps(file_upload_info), 'file_metadata': json.dumps({file_key: [file_metadata]})}) + return Box({ + 'file_upload_info': json.dumps(file_upload_info), + 'file_metadata': json.dumps({file_key: [file_metadata]}) + }) + def response_save_csrf_token(response): """Save CSRF token from response Args: - response (requests.models.Response): response from {host}/accounts/settings/groups/grouplist + response(requests.models.Response): response from {host}/accounts/settings/groups/grouplist Returns: Box: csrf_token @@ -221,20 +234,24 @@ def response_save_csrf_token(response): csrf_token = soup.find(id='csrf_token').get('value') return Box({'csrf_token': csrf_token}) -def response_save_url_described_in_email(response, target, save_key): + +def response_save_url_described_in_email(_, target, save_key): """Extract a URL from an email file and save it in a Box object. Args: - response (requests.models.Response): The response object (not used in this function). - target (str): The target email file name to read from the 'mail' directory. - save_key (str): The key under which the extracted URL will be saved in the Box object. + _(requests.models.Response): The response object (not used in this function). + target(str): The target email file name to read from the 'mail' directory. + save_key(str): The key under which the extracted URL will be saved in the Box object. Returns: Box: A Box object containing the extracted URL under the specified save_key. Raises: ValueError: If no URL is found in the email file. """ - with open('mail/' + target) as f: + target_folder = f'mail/{target}/new' + files = [f for f in os.listdir(target_folder) if os.path.isfile(os.path.join(target_folder, f))] + files.sort(reverse=True) + with open(os.path.join(target_folder, files[0]), 'r') as f: lines = f.readlines() url = '' for line in lines: @@ -246,35 +263,37 @@ def response_save_url_described_in_email(response, target, save_key): raise ValueError(f'No URL found in the email for target: {target}') return Box({save_key: url}) -def response_save_register_data_with_change(response, file_name, key_dict_file, change_type): + +def response_save_register_data_with_change(_, file_name, key_dict_file, change_type): """Save and modify register data from a file based on the specified change type. - + Args: - response (requests.models.Response): response from {host}{next_path} - file_name (str): name of the file containing the data to be registered - key_dict_file (str): name of the file containing the keys for title, resource type, and creator - change_type (int): type of change to apply to the register data + _(requests.models.Response): response from {host}{next_path} + file_name(str): name of the file containing the data to be registered + key_dict_file(str): name of the file containing the keys for title, resource type, and creator + change_type(int): type of change to apply to the register data 1: Title variations in full-width/half-width, uppercase/lowercase, and character form 2: Conversion within the same character type 3: Resource type variations 4: Creator name variations 5: Creator order change 6: Change creator name to 'たかはし さぶろう' - + Returns: Box: register_data - register_data (str): modified register data in JSON format""" + register_data (str): modified register data in JSON format + """ with open('request_params/' + file_name, 'r') as f: register_data = json.load(f) - + with open('request_params/' + key_dict_file, 'r') as f: key_dict = json.load(f) - + # get title title_key = key_dict.get('title') title_key_split = title_key.split('.') title = register_data[title_key_split[0]][int(title_key_split[1])][title_key_split[2]] - + # get resource type resource_type_key = key_dict.get('resource_type') resource_type_key_split = resource_type_key.split('.') @@ -283,7 +302,7 @@ def response_save_register_data_with_change(response, file_name, key_dict_file, # get creator creator_key = key_dict.get('creator') creator = register_data[creator_key] - + if change_type == 1: # Title variations in full-width/half-width, uppercase/lowercase, and character form converted_title_chars = [] @@ -347,27 +366,36 @@ def response_save_register_data_with_change(response, file_name, key_dict_file, elif change_type == 6: # change creator name register_data[creator_key][0]['creatorNames'][0]['creatorName'] = 'たかはし さぶろう' + register_data[creator_key][0]['nameIdentifiers'] = [ + { + "nameIdentifier": "1", + "nameIdentifierScheme": "WEKO" + } + ] + del register_data[creator_key][1] return Box({'register_data': json.dumps(register_data)}) + def response_save_author_search(response): """Save author ID from "{host}/api/author/search"'s response - + Args: - response (requests.models.Response): response from {host}/api/author/search - + response(requests.models.Response): response from {host}/api/author/search + Returns: Box: author_id author_id (str): author ID""" json = response.json() return Box({'author_id': json['hits']['hits'][0]['_id']}) + def response_save_changed_data(response): """Save changed data from "{host}/workflow/activity/detail/{activity_id}?status="'s response Args: - response (requests.models.Response): response from {host}/api/records/{recid} - + response(requests.models.Response): response from {host}/api/records/{recid} + Returns: Box: changed_data changed_data (dict): changed data from the response @@ -376,11 +404,12 @@ def response_save_changed_data(response): invenio_records = soup.find('invenio-records') return Box({'changed_data': invenio_records['record']}) + def response_save_notification_token(response): """Save notification token from response Args: - response (requests.models.Response): response from {host}/account/settings/notifications + response(requests.models.Response): response from {host}/account/settings/notifications Returns: Box: notification_token diff --git a/test/tavern/helper/verify_database_helper.py b/test/tavern/helper/common/verify_database_helper.py similarity index 92% rename from test/tavern/helper/verify_database_helper.py rename to test/tavern/helper/common/verify_database_helper.py index 818fcc3a52..1f0e25d06b 100644 --- a/test/tavern/helper/verify_database_helper.py +++ b/test/tavern/helper/common/verify_database_helper.py @@ -1,29 +1,10 @@ from datetime import datetime, timedelta import json import os -import psycopg2 -from helper.config import DATABASE, REPLACEMENT_DECISION_STRING +from helper.config import REPLACEMENT_DECISION_STRING -def connect_db(): - """Connect to database - - Args: - None - - Returns: - psycopg2.extensions.connection: connection to database - """ - conn = psycopg2.connect( - dbname=DATABASE['dbname'], - user=DATABASE['user'], - password=DATABASE['password'], - host=DATABASE['host'], - port=DATABASE['port'], - ) - return conn - def compare_db_data(cursor, folder_path, replace_params = {}, type_conversion_params = {}, datetime_columns=[]): """Compare data in database with data in excel file diff --git a/test/tavern/helper/config.py b/test/tavern/helper/config.py index 6f6ac4e671..16b1d560eb 100644 --- a/test/tavern/helper/config.py +++ b/test/tavern/helper/config.py @@ -1,32 +1,38 @@ DATABASE = { - 'host': 'localhost', - 'port': 25401, - 'dbname': 'invenio', - 'user': 'invenio', - 'password': 'dbpass123' + 'host': 'localhost', + 'port': 25401, + 'dbname': 'invenio', + 'user': 'invenio', + 'password': 'dbpass123' +} + +REDIS = { + 'host': 'localhost', + 'port': 26301, + 'db': 0 } USERS = { - 'sysadmin': { - 'email': 'wekosoftware@nii.ac.jp', - 'password': 'uspass123' - }, - 'repoadmin': { - 'email': 'repoadmin@example.org', - 'password': 'uspass123' - }, - 'comadmin': { - 'email': 'comadmin@example.org', - 'password': 'uspass123' - }, - 'contributor': { - 'email': 'contributor@example.org', - 'password': 'uspass123' - }, - 'user': { - 'email': 'user@example.org', - 'password': 'uspass123' - } + 'sysadmin': { + 'email': 'wekosoftware@nii.ac.jp', + 'password': 'uspass123' + }, + 'repoadmin': { + 'email': 'repoadmin@example.org', + 'password': 'uspass123' + }, + 'comadmin': { + 'email': 'comadmin@example.org', + 'password': 'uspass123' + }, + 'contributor': { + 'email': 'contributor@example.org', + 'password': 'uspass123' + }, + 'user': { + 'email': 'user@example.org', + 'password': 'uspass123' + } } REPLACEMENT_DECISION_STRING = { @@ -298,3 +304,5 @@ '-NonCommercial-ShareAlike 4.0 International License.' }, ] + +SKIP_HOOK_MARK = 'skip_before_hook' diff --git a/test/tavern/helper/index/data_transformation_helper.py b/test/tavern/helper/index/data_transformation_helper.py new file mode 100644 index 0000000000..eca6ab4abb --- /dev/null +++ b/test/tavern/helper/index/data_transformation_helper.py @@ -0,0 +1,81 @@ +from contextlib import closing +from datetime import datetime + +from psycopg2.extras import RealDictCursor + +from helper.common.connect_helper import connect_db + + +def transform_index_data(index_id): + """Transform index data from the database into the expected format. + + Args: + index_id(int): ID of the index to transform. + + Returns: + dict: Transformed index data. + int: Parent ID of the index. + """ + def get_parents(cursor, index_id): + """Recursively get all parent indices of a given index. + + Args: + cursor(cursor): Database cursor. + index_id(int): Index ID. + + Returns: + list: List of parent indices. + """ + parent_list = [] + if index_id == 0: + return parent_list + cursor.execute("SELECT * FROM index WHERE id = %s;", (index_id,)) + parent = cursor.fetchone() + if parent: + parent_list = get_parents(cursor, parent["parent"]) + parent_list.append(parent) + return parent_list + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + cursor.execute("SELECT * FROM index WHERE id = %s;", (index_id,)) + index_data = cursor.fetchone() + + if index_data is None: + return None + transformd_data = { + "browsing_group": index_data["browsing_group"], + "browsing_role": index_data["browsing_role"], + "children": [], + "cid": index_data["id"], + "contribute_group": index_data["contribute_group"], + "contribute_role": index_data["contribute_role"], + "coverpage_state": index_data["coverpage_state"], + "display_no": index_data["display_no"], + "emitLoadNextLevel": False, + "id": str(index_data["id"]), + "index_link_enabled": index_data["index_link_enabled"], + "is_deleted": index_data["is_deleted"], + "link_name": index_data["index_link_name_english"], + "more_check": index_data["more_check"], + "name": index_data["index_name_english"], + "pid": index_data["parent"], + "position": index_data["position"], + "public_date": datetime.strftime( + index_data["public_date"], "%Y-%m-%dT%H:%M:%S") + if index_data["public_date"] else None, + "public_state": index_data["public_state"], + "recursive_coverpage_check": index_data["recursive_coverpage_check"], + "settings": { + "checked": False, + "isCollapsedOnInit": True + }, + "value": index_data["index_name_english"] + } + if index_data["parent"]: + parents = get_parents(cursor, index_data["parent"]) + parent_id = "" + for p in parents: + parent_id += str(p["id"]) + "/" + transformd_data["parent"] = parent_id[:-1] + return transformd_data diff --git a/test/tavern/helper/index/request_helper.py b/test/tavern/helper/index/request_helper.py new file mode 100644 index 0000000000..eccf871bea --- /dev/null +++ b/test/tavern/helper/index/request_helper.py @@ -0,0 +1,46 @@ +import json +from io import BytesIO + +def generate_request_body( + file_path, + public=None, + harvest=None, + thumbnail=None, + thumbnail_delete=None +): + """Generate a request body by reading a JSON file. + + Args: + file_path(str): The path to the JSON file. + public(int, optional): value to set for "public_state" in the body. + harvest (int, optional): value to set for "harvest_public_state" in the body. + thumbnail (str, optional): value to set for "image_name" in the body. + thumbnail_delete (bool, optional): value to set for "thumbnail_delete_flag" in the body. + + Returns: + Dict[str, Any]: The generated request body as a dictionary. + """ + with open(file_path, "r", encoding="utf-8") as f: + body = json.load(f) + + if public is not None: + body["public_state"] = public + if harvest is not None: + body["harvest_public_state"] = harvest + if thumbnail is not None: + body["image_name"] = thumbnail + if thumbnail_delete is not None: + body["thumbnail_delete_flag"] = thumbnail_delete + return body + + +def generate_no_filename_request(): + """Generate a request body for file upload without a filename. + + Returns: + Dict[str, bytes]: A dictionary containing the file content for upload, without a filename. + """ + file_content = b"Test file content" + file_stream = BytesIO(file_content) + file_stream.name = '' + return {"uploadFile": file_stream.read()} diff --git a/test/tavern/helper/index/response_helper.py b/test/tavern/helper/index/response_helper.py new file mode 100644 index 0000000000..c90668a30c --- /dev/null +++ b/test/tavern/helper/index/response_helper.py @@ -0,0 +1,231 @@ +from contextlib import closing +from datetime import datetime, timezone +import json + +from box import Box +import dill +from psycopg2.extras import RealDictCursor + +from helper.common.connect_helper import connect_db, connect_redis + + +def set_pdfcoverpage(response, avail, lang, map=None): + """Set the availability of PDF cover page and update session language. + + Args: + response(Response): The response object from which to extract cookies. + avail(str): The availability status to set for PDF cover page. + lang(str): The language to set in the session. + map(str, optional): determine the mapping for group settings. + + Returns: + Box: A Box object containing role IDs if map is "delete", otherwise empty. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cur: + update_query = "UPDATE pdfcoverpage_set SET avail = %s;" + cur.execute(update_query, (avail,)) + conn.commit() + + cookie = response.cookies.get_dict() + session_key = cookie.get('session', '').split(".")[0] + + # Connect to Redis and update the session with the new language settings + redis_1 = connect_redis(db=1) + session = dill.loads(redis_1.get(session_key)) + session["language"] = str(lang) + session["selected_language"] = str(lang) + redis_1.set(session_key, dill.dumps(session, protocol=4)) + + role_ids = [] + if map == "register": + redis_4 = connect_redis(db=4) + key = "weko3_example_org_gakunin_groups" + value = { + "updated_at": datetime.now(timezone.utc).isoformat(timespec='seconds'), + "groups": "jc_weko3_example_org_groups_add1," + "jc_weko3_example_org_groups_add2,jc_weko3_example_org_groups_add3" + } + redis_4.hset(key, mapping=value) + elif map == "delete": + redis_4 = connect_redis(db=4) + key = "weko3_example_org_gakunin_groups" + value = { + "updated_at": datetime.now(timezone.utc).isoformat(timespec='seconds'), + "groups": "" + } + redis_4.hset(key, mapping=value) + + insert_queries = [ + "INSERT INTO accounts_role(name) VALUES ('jc_weko3_example_org_groups_delete1');", + "INSERT INTO accounts_role(name) VALUES ('jc_weko3_example_org_groups_delete2');", + "INSERT INTO accounts_role(name) VALUES ('jc_weko3_example_org_groups_delete3');" + ] + for query in insert_queries: + cur.execute(query) + conn.commit() + search_query = "SELECT id FROM accounts_role WHERE name IN " \ + "('jc_weko3_example_org_groups_delete1', 'jc_weko3_example_org_groups_delete2', 'jc_weko3_example_org_groups_delete3');" + cur.execute(search_query) + role_ids = [row[0] for row in cur.fetchall()] + index_updatequery = "UPDATE index SET browsing_role = browsing_role || %s, contribute_role = contribute_role || %s;" + role_ids_str = ",".join(str(role_id) for role_id in role_ids) + cur.execute(index_updatequery, ("," + role_ids_str, "," + role_ids_str)) + conn.commit() + return Box({ + "role_ids": role_ids + }) + + +def count_indices(_): + """Count the number of indices in the database. + + Args: + _ (Response): Unused parameter for compatibility. + + Returns: + Box: A Box object containing the count of indices. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cursor: + cursor.execute("SELECT COUNT(*) FROM index;") + count = cursor.fetchone()[0] + return Box({"index_count": count}) + + +def set_lock_index(_, index_id, edit=False): + """Set a lock on an index and optionally retrieve its details. + + Args: + _ (Response): Unused parameter for compatibility. + index_id (int): The ID of the index to lock. + edit (bool, optional): Boolean flag indicating whether to retrieve index details. + + Returns: + Box: A Box object containing index details if edit is True, otherwise the count of indices. + """ + redis_0 = connect_redis() + key = f"lock_index_{index_id}" + redis_0.set(key, "1") + if edit: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + cursor.execute("SELECT * FROM index WHERE id = %s;", (index_id,)) + index = cursor.fetchone() + + index_serializable = { + key: value.isoformat() if isinstance(value, datetime) else value + for key, value in index.items() + } + return Box({ + "index": json.dumps(index_serializable), + }) + else: + return count_indices(_) + + +def set_target_id(_, file_path, invalid=False): + """Set the target ID from a file and optionally retrieve its details. + + Args: + _ (Response): Unused parameter for compatibility. + file_path (str): The path to the file containing target information. + invalid (bool, optional): Boolean flag indicating whether to retrieve index details. + + Returns: + Box: A Box object containing target ID list and index details if invalid is True. + """ + with open(file_path, "r", encoding="utf-8") as f: + target_info = json.load(f) + target_id_list = [target_info.get("id")] + if invalid: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + cursor.execute("SELECT * FROM index WHERE id = %s;", (target_id_list[0],)) + index = cursor.fetchone() + + index_serializable = { + key: value.isoformat() if isinstance(value, datetime) else value + for key, value in index.items() + } + return Box({ + "target_id_list": target_id_list, + "index": json.dumps(index_serializable), + }) + else: + return Box({ + "target_id_list": target_id_list, + }) + + +def get_indices(_): + """Retrieve all indices from the database and return them as a JSON string. + + Args: + _ (Response): Unused parameter for compatibility. + + Returns: + Box: A Box object containing the list of indices as a JSON string. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + cursor.execute("SELECT * FROM index ORDER BY id;") + rows = cursor.fetchall() + + index_list = [] + for row in rows: + index_serializable = { + key: value.isoformat() if isinstance(value, datetime) else value + for key, value in row.items() + } + index_list.append(index_serializable) + return Box({ + "index_list": json.dumps(index_list), + }) + + +def set_thumbnail_path(response): + """Extract thumbnail path from the response JSON. + + Args: + response (Response): The response object containing JSON data. + + Returns: + Box: A Box object containing the thumbnail path. + """ + response_data = response.json() + return Box({"thumbnail_path": response_data.get("data", {}).get("path", "")}) + + +def set_harvest(_, index_id): + """Set harvest settings for a given index ID. + + Args: + _ (Response): Unused parameter for compatibility. + index_id (int): The ID of the index to set harvest settings for. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + insert_query = "INSERT INTO harvest_settings" \ + "(repository_name, base_url, metadata_prefix, index_id, update_style, auto_distribution)"\ + " VALUES (%s, %s, %s, %s, %s, %s);" + cursor.execute( + insert_query, + ("Test Repository", "http://example.com/oai", "oai_dc", index_id, "1", "1")) + conn.commit() + + +def set_index_view_permission(_, index_id, role_id=""): + """Set the browsing role for a given index ID. + + Args: + _ (Response): Unused parameter for compatibility. + index_id (int): The ID of the index to set the browsing role for. + role_id (str, optional): The ID of the role to set as the browsing role. + If empty, it will clear the browsing role. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + update_query = "UPDATE index SET browsing_role = %s WHERE id = %s;" + cursor.execute(update_query, (role_id, index_id)) + conn.commit() diff --git a/test/tavern/helper/index/verify_helper.py b/test/tavern/helper/index/verify_helper.py new file mode 100644 index 0000000000..1d2abbeea1 --- /dev/null +++ b/test/tavern/helper/index/verify_helper.py @@ -0,0 +1,1302 @@ +from contextlib import closing +from datetime import datetime, timezone +from time import sleep +from urllib.parse import urlencode +import json + +import requests +from box import Box +from bs4 import BeautifulSoup +from psycopg2.extras import RealDictCursor + +from helper.common.connect_helper import connect_db +from helper.index.data_transformation_helper import transform_index_data + + +def verify_indexedit_elements( + response, + lang, + coverpage, + map_groups={}, + nodeid="0" + ): + """Verify elements in index edit page + + Args: + response(Response): Response object from the index edit page request + lang(str): Expected language code + coverpage(str): Expected coverpage setting value + map_groups(dict, optional): Expected map groups. Defaults to {}. + nodeid(str, optional): Expected node ID. Defaults to "0". + """ + soup = BeautifulSoup(response.text, 'html.parser') + + # lang-code + lang_code = soup.find(id='lang-code').get('value') + assert lang_code == lang + + # get_tree_json + get_tree_json = soup.find(id='get_tree_json').text + assert get_tree_json == '/api/tree' + + # upt_tree_json + upt_tree_json = soup.find(id='upt_tree_json').text + assert upt_tree_json == '' + + # mod_tree_detail + mod_tree_detail = soup.find(id='mod_tree_detail').text + assert mod_tree_detail == '/api/tree/index/' + + # admin_coverpage_setting + admin_coverpage_setting = soup.find(id='admin_coverpage_setting').text + assert admin_coverpage_setting == str(coverpage) + + # show_modal + show_modal = soup.find(id='show_modal').get('value') + assert show_modal == 'False' + + # app-root-tree-hensyu + app_root_tree_hensyu = soup.find('app-root-tree-hensyu') + assert app_root_tree_hensyu['nodeid'] == nodeid + + if map_groups: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + if map_groups.get("add"): + role_query = "SELECT id FROM accounts_role WHERE name IN %s;" + cursor.execute(role_query, (tuple(map_groups["add"]),)) + added_roles = cursor.fetchall() + index_query = "SELECT browsing_role, contribute_role FROM index;" + cursor.execute(index_query) + index_roles = cursor.fetchall() + for index in index_roles: + browsing_role_ids = index["browsing_role"].split(',') + contribute_role_ids = index["contribute_role"].split(',') + added_roles_ids = [str(role["id"]) for role in added_roles] + assert all(role_id in browsing_role_ids for role_id in added_roles_ids) + assert all(role_id in contribute_role_ids for role_id in added_roles_ids) + elif map_groups.get("delete"): + index_query = "SELECT browsing_role, contribute_role FROM index;" + cursor.execute(index_query) + index_roles = cursor.fetchall() + for index in index_roles: + browsing_role_ids = index["browsing_role"].split(',') + contribute_role_ids = index["contribute_role"].split(',') + deleted_roles_ids = json.loads(map_groups["delete"]) + assert all(str(role_id) not in browsing_role_ids for role_id in deleted_roles_ids) + assert all(str(role_id) not in contribute_role_ids for role_id in deleted_roles_ids) + + +def verify_index_tree( + response, + expected_tree_file, + role_id, + edited_id_list=None, + deleted_id_list=None, + no_action=False + ): + """Verify the index tree structure in the response. + + Args: + response(Response): Response object from the index tree request. + expected_tree_file(str): Path to the file containing the expected index tree structure. + role_id(str): Role ID to filter the tree information. + edited_id_list(list, optional): List of IDs of the edited index nodes. Defaults to None. + deleted_id_list(list, optional): List of IDs of the deleted index nodes. Defaults to None. + no_action(bool, optional): Flag to indicate no action. Defaults to False. + + Raises: + ValueError: If an edited index ID is not found in the database. + """ + def get_browsing_treeinfo(tree, role_id): + """Recursively get the browsing tree information for a given role ID. + + Args: + tree(list): Current tree structure. + role_id(str): Role ID to filter the tree information. + + Returns: + list: Filtered tree structure based on the browsing role. + """ + hide_keys = ["browsing_group", "browsing_role", "contribute_group", "contribute_role", "public_date", "public_state"] + browsing_info = [] + for node in tree: + public_state = node.get("public_state", "") + browsing_role = node.get("browsing_role", "") + for key in hide_keys: + if key in node: + del node[key] + if public_state and str(role_id) in browsing_role.split(','): + node["children"] = get_browsing_treeinfo(node.get("children", []), role_id) + browsing_info.append(node) + if node.get("id") == "more": + browsing_info.append(node) + return browsing_info + + def upsert_node(tree, node_json, my_id, parent_id): + """Upsert a node in the tree structure. + + Args: + tree(list): Current tree structure. + node_json(dict): Node data to upsert. + my_id(str): ID of the node to upsert. + parent_id(int): Parent ID of the node. + """ + if parent_id == 0: + for idx, node in enumerate(tree): + if node["id"] == my_id: + tree[idx] = node_json + return + tree.append(node_json) + return + for node in tree: + if node["id"] == "more": + continue + if node["id"] == str(parent_id): + for idx, child in enumerate(node["children"]): + if child["id"] == str(my_id): + node_json["children"] = child.get("children", []) + node["children"][idx] = node_json + return + node["children"].append(node_json) + return + upsert_node(node["children"], node_json, my_id, parent_id) + def delete_node(tree, my_id): + """Delete a node from the tree structure. + + Args: + tree(list): Current tree structure. + my_id(str): ID of the node to delete. + """ + for idx, node in enumerate(tree): + if node["id"] == "more": + continue + if node["id"] == str(my_id): + del tree[idx] + return + delete_node(node["children"], my_id) + + with open(expected_tree_file, 'r') as f: + expected_tree = json.load(f) + if role_id not in [1, 2] and not no_action: + browsing_tree = get_browsing_treeinfo(expected_tree, role_id) + else: + browsing_tree = expected_tree + actual_tree = response.json() + + if edited_id_list: + edited_id_list = json.loads(str(edited_id_list)) + for edited_id in edited_id_list: + tree_json = transform_index_data(edited_id) + if tree_json is None: + raise ValueError(f"Index with ID {edited_id} not found in the database.") + index_id = tree_json["id"] + parent_id = tree_json["pid"] + upsert_node(browsing_tree, tree_json, index_id, parent_id) + + if deleted_id_list: + deleted_id_list = json.loads(str(deleted_id_list)) + for deleted_id in deleted_id_list: + delete_node(browsing_tree, deleted_id) + + try: + assert actual_tree == browsing_tree + except AssertionError as e: + print("Actual Tree:", actual_tree) + print("Expected Tree:", browsing_tree) + raise e + + +def verify_index_info(response, id, within_community=True): + """Verify the index information in the response. + + Args: + response(Response): Response object from the index info request. + id(int): ID of the index to verify. + within_community(bool, optional): Whether to verify within the community context. Defaults to True. + """ + def is_gakunin_group(name): + """Check if the role name is a Gakunin group. + + Args: + name(str): Role name. + + Returns: + bool: True if it's a Gakunin group, False otherwise. + """ + return name.startswith("jc_") and name.find("_groups") != -1 + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT * FROM index WHERE id = %s;" + role_query = "SELECT id, name FROM accounts_role WHERE name NOT IN ('System Administrator', 'Repository Administrator');" + cursor.execute(index_query, (id,)) + index_data = cursor.fetchone() + cursor.execute(role_query) + roles = cursor.fetchall() + + roles.append({"id": -98, "name": "Authenticated User"}) + roles.append({"id": -99, "name": "Guest"}) + + response_data = response.json() + + for k in response_data: + if k == "can_edit": + assert response_data[k] == within_community + elif k == "have_children": + children_query = "SELECT id FROM index WHERE parent = %s;" + cursor.execute(children_query, (id,)) + children = cursor.fetchall() + assert response_data[k] == (len(children) > 0) + elif k in ["browsing_role", "contribute_role"]: + expected_roles = { + "allow": [], + "deny": [] + } + + splited_roles = index_data[k].split(',') + for r in roles: + if is_gakunin_group(r["name"]): + continue + if str(r["id"]) in splited_roles: + expected_roles["allow"].append(dict(r)) + else: + expected_roles["deny"].append(dict(r)) + + assert sorted(response_data[k]["allow"], key=lambda x: x["id"])\ + == sorted(expected_roles["allow"], key=lambda x: x["id"]) + assert sorted(response_data[k]["deny"], key=lambda x: x["id"])\ + == sorted(expected_roles["deny"], key=lambda x: x["id"]) + elif k in ["browsing_group", "contribute_group"]: + expected_groups = { + "allow": [], + "deny": [] + } + + splited_groups = index_data[k].split(',') + for r in roles: + if is_gakunin_group(r["name"]): + if str(r["id"]) in splited_groups: + expected_groups["allow"].append({ + "id": str(r["id"]) + "gr", + "name": r["name"] + }) + else: + expected_groups["deny"].append({ + "id": str(r["id"]) + "gr", + "name": r["name"] + }) + + assert sorted(response_data[k]["allow"], key=lambda x: x["id"])\ + == sorted(expected_groups["allow"], key=lambda x: x["id"]) + assert sorted(response_data[k]["deny"], key=lambda x: x["id"])\ + == sorted(expected_groups["deny"], key=lambda x: x["id"]) + else: + assert response_data[k] == (index_data[k] if index_data[k] is not None else "") + + +def verify_created_index(response, id, name, user_id, map=False, parent_id=0): + """Verify the created index information in the response and database. + + Args: + response(Response): Response object from the index creation request. + id(int): ID of the created index. + name(str): Name of the created index. + user_id(int): User ID of the owner of the created index. + map(bool, optional): Whether the index is created through map. Defaults to False. + parent_id(int, optional): Parent ID of the created index. Defaults to 0. + """ + expected_response = { + "status": 201, + "message": "Index created successfully.", + "errors": [] + } + assert response.json() == expected_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + query = "SELECT * FROM index WHERE id = %s;" + cursor.execute(query, (id,)) + index_data = cursor.fetchone() + + with open('response_data/index/created_index.json', 'r') as f: + expected_data = json.load(f) + + now = datetime.now(timezone.utc).replace(tzinfo=None) + expected_data["created"] = now + expected_data["updated"] = now + expected_data["id"] = id + expected_data["index_name"] = name + expected_data["index_name_english"] = name + expected_data["index_link_name_english"] = name + expected_data["owner_user_id"] = user_id + + if map: + roles_query = "SELECT id, name FROM accounts_role WHERE name like 'jc_%';" + cursor.execute(roles_query) + roles = cursor.fetchall() + role_ids_str = ",".join(str(role["id"]) for role in roles) + expected_data["browsing_role"] = "1,2,3,4," + role_ids_str + ",-98,-99" + expected_data["contribute_role"] = "1,2,3,4," + role_ids_str + ",-98,-99" + + if parent_id: + select_query = "SELECT * FROM index WHERE id = %s;" + count_query = "SELECT COUNT(*) FROM index WHERE parent = %s;" + cursor.execute(select_query, (parent_id,)) + parent_data = cursor.fetchone() + cursor.execute(count_query, (parent_id,)) + count = cursor.fetchone()["count"] - 1 + expected_data["parent"] = parent_id + expected_data["position"] = count + expected_data["harvest_public_state"] = parent_data["harvest_public_state"] + expected_data["display_format"] = parent_data["display_format"] + if parent_data["recursive_public_state"]: + expected_data["public_state"] = parent_data["public_state"] + expected_data["public_date"] = parent_data["public_date"] + expected_data["recursive_public_state"] = True + if parent_data["recursive_browsing_role"]: + expected_data["browsing_role"] = parent_data["browsing_role"] + expected_data["recursive_browsing_role"] = True + if parent_data["recursive_contribute_role"]: + expected_data["contribute_role"] = parent_data["contribute_role"] + expected_data["recursive_contribute_role"] = True + if parent_data["recursive_browsing_group"]: + expected_data["browsing_group"] = parent_data["browsing_group"] + expected_data["recursive_browsing_group"] = True + if parent_data["recursive_contribute_group"]: + expected_data["contribute_group"] = parent_data["contribute_group"] + expected_data["recursive_contribute_group"] = True + + for k in index_data: + print(f"{k}: {index_data[k]} (expected: {expected_data.get(k)})") + if k in ["created", "updated"]: + assert abs((index_data[k] - expected_data[k]).total_seconds()) < 1 + else: + assert index_data[k] == expected_data[k] + + +def verify_edited_index( + response, + file_path, + public=None, + harvest=None, + thumbnail="", + image_delete=False, + oai_action="" + ): + """Verify the edited index information in the response and database. + + Args: + response(Response): Response object from the index edit request. + file_path(str): Path to the file containing the expected index data. + public(bool, optional): Expected public state. Defaults to None. + harvest(bool, optional): Expected harvest state. Defaults to None. + thumbnail(str, optional): Expected thumbnail image name. Defaults to "". + image_delete(bool, optional): Whether the image was deleted. Defaults to False. + oai_action(str, optional): Expected OAI action. Defaults to "". + """ + def get_children(cursor, parent_id): + """Recursively get all children of a given parent index. + + Args: + cursor(cursor): Database cursor. + parent_id(int): Parent index ID. + + Returns: + list: List of child indices. + """ + cursor.execute("SELECT * FROM index WHERE parent = %s;", (parent_id,)) + children = cursor.fetchall() + all_children = [] + for child in children: + all_children.append(child) + all_children.extend(get_children(cursor, child["id"])) + return all_children + + def get_roles(group_dict, role_dict): + """Extract allowed role IDs from group and role dictionaries. + + Args: + group_dict(dict): Dictionary containing group information. + role_dict(dict): Dictionary containing role information. + + Returns: + list: List of allowed role IDs. + """ + allow_roles = [role["id"] for role in role_dict.get("allow", [])] + allow_roles.extend([int(gr["id"][:-2]) for gr in group_dict.get("allow", []) + if gr["id"].endswith("gr")]) + return allow_roles + + def get_parents(cursor, index_id): + """Recursively get all parent indices of a given index. + + Args: + cursor(cursor): Database cursor. + index_id(int): Index ID. + + Returns: + list: List of parent indices. + """ + parent_list = [] + if index_id == 0: + return parent_list + cursor.execute("SELECT * FROM index WHERE id = %s;", (index_id,)) + parent = cursor.fetchone() + if parent: + parent_list = get_parents(cursor, parent["parent"]) + parent_list.append(parent) + return parent_list + + expected_response = { + "status": 200, + "message": "Index updated successfully.", + "errors": [], + "delete_flag": image_delete + } + assert response.json() == expected_response + + with open(file_path, "r", encoding="utf-8") as f: + expected_data = json.load(f) + + expected_data["image_name"] = thumbnail + + target_id = expected_data["id"] + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + cursor.execute("SELECT * FROM index WHERE id = %s;", (target_id,)) + index_data = cursor.fetchone() + recursive_list = get_children(cursor, target_id) + roles_query = "SELECT id, name FROM accounts_role;" + cursor.execute(roles_query) + roles = cursor.fetchall() + roles_dict = {str(r["id"]): r["name"] for r in roles} + roles_dict["-98"] = "Authenticated User" + roles_dict["-99"] = "Guest" + + for k in expected_data: + if k in ["id", "parent", "position", "can_edit", "have_children"]: + continue + if k in ["browsing_group", "contribute_group"]: + assert index_data[k] == "" + elif k == "biblio_flag": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + assert rid["online_issn"] == index_data["online_issn"] + else: + pass + elif k == "browsing_role": + expected_roles = get_roles(expected_data["browsing_group"], expected_data["browsing_role"]) + splited_roles = index_data[k].split(',') + assert sorted([int(r) for r in splited_roles if r]) == sorted(expected_roles) + elif k == "contribute_role": + expected_roles = get_roles(expected_data["contribute_group"], expected_data["contribute_role"]) + splited_roles = index_data[k].split(',') + assert sorted([int(r) for r in splited_roles if r]) == sorted(expected_roles) + elif k == "recursive_browsing_group": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + assert rid["browsing_group"] == index_data["browsing_group"] + expected_gakunin_groups = [gr for gr in expected_data["browsing_group"].get("allow", []) + if gr["id"].endswith("gr")] + splited_roles = rid["browsing_role"].split(',') + for gr in expected_gakunin_groups: + assert gr["id"][:-2] in splited_roles + else: + pass + elif k == "recursive_browsing_role": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + rid_splited_roles = rid["browsing_role"].split(',') + index_data_splited_roles = index_data["browsing_role"].split(',') + for role in index_data_splited_roles: + if roles_dict.get(role).startswith("jc_") and roles_dict.get(role).find("_groups") != -1: + assert role not in rid_splited_roles + else: + assert role in rid_splited_roles + else: + pass + elif k == "recursive_contribute_group": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + assert rid["contribute_group"] == index_data["contribute_group"] + expected_gakunin_groups = [gr for gr in expected_data["contribute_group"].get("allow", []) + if gr["id"].endswith("gr")] + splited_roles = rid["contribute_role"].split(',') + for gr in expected_gakunin_groups: + assert gr["id"][:-2] in splited_roles + else: + pass + elif k == "recursive_contribute_role": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + rid_splited_roles = rid["contribute_role"].split(',') + index_data_splited_roles = index_data["contribute_role"].split(',') + for role in index_data_splited_roles: + if roles_dict.get(role).startswith("jc_") and roles_dict.get(role).find("_groups") != -1: + assert role not in rid_splited_roles + else: + assert role in rid_splited_roles + else: + pass + elif k == "recursive_coverpage_check": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + assert rid["coverpage_state"] == index_data["coverpage_state"] + else: + pass + elif k == "recursive_public_state": + assert index_data[k] == False + for rid in recursive_list: + if expected_data[k]: + assert rid["public_state"] == index_data["public_state"] + assert rid["public_date"] == index_data["public_date"] + else: + pass + elif k == "thumbnail_delete_flag": + if image_delete: + assert index_data["image_name"] == "" + else: + assert index_data["image_name"] == expected_data["image_name"] + elif k == "public_date": + expected_date = None + if expected_data[k]: + expected_date = datetime.fromisoformat(expected_data[k]) + assert index_data[k] == expected_date + elif k == "public_state": + if public is not None: + assert index_data[k] == public + else: + assert index_data[k] == expected_data[k] + elif k == "harvest_public_state": + if harvest is not None: + assert index_data[k] == harvest + else: + assert index_data[k] == expected_data[k] + else: + assert index_data[k] == expected_data[k] + + if public and harvest and expected_data["public_state"] and expected_data["harvest_public_state"]: + root = get_parents(cursor, target_id) + spec = "" + description = "" + for r in root: + spec += f"{r['id']}:" + description += f"{r['index_name_english']}->" + spec = spec[:-1] + if index_data["parent"] == 0: + description = index_data["index_name"] + else: + description = description[:-2] + + now = datetime.now(timezone.utc).replace(tzinfo=None) + expected_oai_set = { + "created": now, + "updated": now, + "id": target_id, + "spec": spec, + "name": index_data["index_name"], + "description": description, + "search_pattern": f'path:"{target_id}"' + } + + oai_query = "SELECT * FROM oaiserver_set WHERE id = %s;" + for attempt in range(10): + cursor.execute(oai_query, (target_id,)) + oai_data = cursor.fetchone() + if oai_data: + break + if attempt == 9: + raise AssertionError("OAI Set data not found after multiple attempts.") + sleep(1) + + for k in expected_oai_set: + if k in ["created", "updated"]: + if image_delete: + continue + assert abs((oai_data[k] - expected_oai_set[k]).total_seconds()) < 1 + else: + assert oai_data[k] == expected_oai_set[k] + + if oai_action: + if oai_action == "edit": + root = get_parents(cursor, target_id) + spec = "" + description = "" + for r in root: + spec += f"{r['id']}:" + description += f"{r['index_name_english']}->" + spec = spec[:-1] + if index_data["parent"] == 0: + description = index_data["index_name"] + else: + description = description[:-2] + + expected_oai_set = { + "id": target_id, + "spec": spec, + "name": index_data["index_name"], + "description": description, + "search_pattern": f'path:"{target_id}"' + } + + oai_query = "SELECT * FROM oaiserver_set WHERE id = %s;" + for attempt in range(10): + cursor.execute(oai_query, (target_id,)) + oai_data = cursor.fetchone() + if oai_data: + if oai_data["created"] != oai_data["updated"]: + break + if attempt == 9: + raise AssertionError("OAI Set data not found after multiple attempts.") + sleep(1) + for k in expected_oai_set: + assert oai_data[k] == expected_oai_set[k] + elif oai_action == "delete": + oai_query = "SELECT * FROM oaiserver_set WHERE id = %s;" + for attempt in range(10): + cursor.execute(oai_query, (target_id,)) + oai_data = cursor.fetchone() + if oai_data is None: + break + if attempt == 9: + raise AssertionError("OAI Set data not found after multiple attempts.") + sleep(1) + assert oai_data is None + + +def verify_upload_thumbnail(response, file_name): + """Verify the upload thumbnail response. + + Args: + response(Response): Response object from the upload thumbnail request. + file_name(str): Expected file name of the uploaded thumbnail. + """ + expected_response = { + "code": 0, + "data": { + "path": f"/data/indextree/{file_name}" + }, + "msg": "file upload success" + } + assert response.json() == expected_response + + +def verify_deleted_index(response, deleted_ids, single_item_ids=[], multi_item_ids=[]): + """Verify the deleted index information in the response and database. + + Args: + response(Response): Response object from the index deletion request. + deleted_ids(list): List of IDs of the deleted indices. + single_item_ids(list, optional): List of single item IDs associated with the deleted index. Defaults to []. + multi_item_ids(list, optional): List of multi item IDs associated with the deleted index. Defaults to []. + """ + expected_response = { + "status": 200, + "message": "Index deleted successfully.", + "errors": [] + } + assert response.json() == expected_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT is_deleted FROM index WHERE id IN %s;" + cursor.execute(index_query, (tuple(deleted_ids),)) + index_data = cursor.fetchall() + for data in index_data: + assert data["is_deleted"] == True + + oai_query = "SELECT * FROM oaiserver_set WHERE id IN %s;" + for i in range(10): + cursor.execute(oai_query, (tuple(deleted_ids),)) + oai_data = cursor.fetchone() + if oai_data is None: + break + if i == 9: + raise AssertionError("OAI Set data not found after multiple attempts.") + sleep(1) + assert oai_data is None + + deleted_pid_query = "SELECT status FROM pidstore_pid " \ + "WHERE pid_type = 'recid' AND pid_value IN %s;" + if single_item_ids: + cursor.execute(deleted_pid_query, (tuple([str(id) for id in single_item_ids]),)) + pid_data = cursor.fetchall() + for pid in pid_data: + assert pid["status"] == "D" + updated_pid_query = "SELECT object_uuid FROM pidstore_pid " \ + "WHERE pid_type = 'recid' AND pid_value = %s;" + records_query = "SELECT json FROM records_metadata WHERE id = %s;" + for item_id in multi_item_ids: + cursor.execute(updated_pid_query, (str(item_id),)) + pid_data = cursor.fetchone() + assert pid_data is not None + object_uuid = pid_data["object_uuid"] + cursor.execute(records_query, (object_uuid,)) + record_data = cursor.fetchone() + record_json = record_data["json"] + for deleted_id in deleted_ids: + assert str(deleted_id) not in record_json.get("path", []) + + +def verify_not_deleted_index(response, target_ids, item_ids): + """Verify that the index is not deleted in the response and database. + + Args: + response(Response): Response object from the index deletion request. + target_ids(list): List of target index IDs. + item_ids(list): List of item IDs associated with the target indices. + """ + expected_response = { + "status": 200, + "message": "Index deleted successfully.", + "errors": [] + } + assert response.json() == expected_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT is_deleted FROM index WHERE id IN %s;" + cursor.execute(index_query, (tuple(target_ids),)) + index_data = cursor.fetchall() + for data in index_data: + assert data["is_deleted"] == False + + pid_query = "SELECT status FROM pidstore_pid WHERE pid_type = 'recid' AND pid_value IN %s;" + cursor.execute(pid_query, (tuple([str(id) for id in item_ids]),)) + pid_data = cursor.fetchall() + for pid in pid_data: + assert pid["status"] == "R" + + +def verify_delete_not_exist_index(response): + """Verify that the response indicates the index deletion failed due to non-existence. + + Args: + response(Response): Response object from the index deletion request. + """ + expected_response = { + "status": 200, + "message": "Failed to delete index.", + "errors": [] + } + assert response.json() == expected_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT is_deleted FROM index;" + cursor.execute(index_query) + index_data = cursor.fetchall() + for data in index_data: + assert data["is_deleted"] == False + + +def verify_private_with_doi(response, index): + """Verify that the response indicates the index cannot be private due to DOI links. + + Args: + response(Response): Response object from the request. + index(dict): Expected index data. + """ + expected_response = { + "status": 200, + "message": "", + "errors": [ + "The index cannot be kept private because there are links from items that have a DOI." + ], + "delete_flag": False + } + assert response.json() == expected_response + + verify_index(index) + + +def verify_harvest_private_with_doi(response, index): + """Verify that the response indicates the index harvest cannot be private due to DOI links. + + Args: + response(Response): Response object from the request. + index(dict): Expected index data. + """ + expected_response = { + "status": 200, + "message": "", + "errors": [ + "Index harvests cannot be kept private because there are links from items that have a DOI." + ], + "delete_flag": False + } + assert response.json() == expected_response + + verify_index(index) + + +def verify_index_locked(response, count=None, index=None, is_edit=False): + """Verify that the response indicates the index is locked and the count of indices remains unchanged. + + Args: + response(Response): Response object from the request. + count(int, optional): Expected count of indices. + index(dict, optional): Expected index data. + is_edit(bool, optional): Whether the operation is an edit. Defaults to False. + """ + expected_response = { + "status": 200, + "message": "", + "errors": [ + "Index Delete is in progress on another device." + ] + } + if is_edit: + expected_response["delete_flag"] = False + assert response.json() == expected_response + + if count is not None: + verify_count(count) + if index is not None: + verify_index(index) + + +def verify_edit_during_import(response, index): + """Verify that the response indicates the index cannot be edited due to an import in progress. + + Args: + response(Response): Response object from the index edit request. + index(dict): Expected index data. + """ + try: + expected_response = { + "status": 200, + "message": "", + "errors": [ + "The index cannot be updated becase import is in progress." + ], + "delete_flag": False + } + assert response.json() == expected_response + + verify_index(index) + except AssertionError as e: + print("Response JSON:", response.json()) + raise e + + +def verify_delete_with_doi(response, delete_id): + """Verify that the response indicates the index cannot be deleted due to DOI links. + + Args: + response(Response): Response object from the request. + delete_id(int): ID of the index attempted to be deleted. + """ + excepted_response = { + "status": 200, + "message": "", + "errors": [ + "The index cannot be deleted because there is a link from an item that has a DOI." + ] + } + assert response.json() == excepted_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + query = "SELECT is_deleted FROM index WHERE id = %s;" + cursor.execute(query, (delete_id,)) + index_data = cursor.fetchone() + assert index_data["is_deleted"] == False + + +def verify_delete_with_editing(response, delete_id): + """Verify that the response indicates the index cannot be deleted due to being edited. + + Args: + response(Response): Response object from the request. + delete_id(int): ID of the index attempted to be deleted. + """ + excepted_response = { + "status": 200, + "message": "", + "errors": [ + "This index cannot be deleted because the item belonging to this index is being edited." + ] + } + assert response.json() == excepted_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + query = "SELECT is_deleted FROM index WHERE id = %s;" + cursor.execute(query, (delete_id,)) + index_data = cursor.fetchone() + assert index_data["is_deleted"] == False + + +def verify_delete_with_harvesting(response, delete_id): + """Verify that the response indicates the index cannot be deleted due to being harvested. + + Args: + response(Response): Response object from the request. + delete_id(int): ID of the index attempted to be deleted. + """ + excepted_response = { + "status": 200, + "message": "", + "errors": [ + "The index cannot be deleted becase the index in harvester settings." + ] + } + assert response.json() == excepted_response + + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + query = "SELECT is_deleted FROM index WHERE id = %s;" + cursor.execute(query, (delete_id,)) + index_data = cursor.fetchone() + assert index_data["is_deleted"] == False + + +def verify_delete_no_specify_index(_, indices): + """Verify that no indices are deleted. + + Args: + _(Response): Response object from the request. + indices(list): Expected list of indices. + """ + verify_indices(indices) + + +def verify_no_filename(response, path, code, expected_location=None, host=None): + """Verify that the response indicates a bad request due to no filename. + + Args: + response(Response): Response object from the initial request. + path(str): Path to the file to be uploaded. + code(int): Expected status code of the response. + expected_location(str, optional): Expected value of the Location header for redirection. Defaults to None. + host(str, optional): Host URL for constructing the expected Location header. Defaults to None. + """ + target_url = response.url + 'admin/indexedit/upload' + cookies = response.cookies + with open(path, "rb") as f: + files = { + "uploadFile": ('', f, 'multipart/form-data'), + } + r = requests.post(target_url, files=files, verify=False, cookies=cookies, allow_redirects=False) + assert r.status_code == code + soup = BeautifulSoup(r.text, 'html.parser') + h1 = soup.find('h1') + if code == 400: + assert h1.text == "Bad Request" + elif code == 403: + assert h1.text.endswith("Permission required") + elif code == 302 and expected_location is not None and host is not None: + verify_location_header(r, expected_location, host) + +def verify_location_header(response, expected_location, host): + """Verify that the response contains the expected Location header. + + Args: + response(Response): The response object to check. + expected_location(str): The expected value of the Location header. + host(str): The host URL + """ + location_header = response.headers.get('Location') + encoded_expected_location = urlencode({'next': expected_location}) + full_expected_location = f"{host}/login/?{encoded_expected_location}" + assert location_header == full_expected_location + + +def verify_bad_request(response, message, count=None, index=None, indices=None): + """Verify that the response indicates a bad request with the expected message. + + Args: + response(Response): Response object from the request. + message(str, optional): Expected error message. + count(int, optional): Expected count of indices. + index(dict, optional): Expected index data. + indices(list, optional): Expected list of indices. + """ + expected_response = { + "message": message, + "status": 400, + } + assert response.json() == expected_response + + if count is not None: + verify_count(count) + if index is not None: + verify_index(index) + if indices is not None: + verify_indices(indices) + + +def verify_bad_request_html(response): + """Verify that the response indicates a bad request in HTML format. + + Args: + response(Response): Response object from the request. + """ + soup = BeautifulSoup(response.text, 'html.parser') + h1 = soup.find('h1') + assert h1.text == "Bad Request" + + +def verify_unauthorized(response, count=None, index=None, indices=None, target_ids=None, item_ids=None): + """Verify that the response indicates an unauthorized access with the expected message. + + Args: + response(Response): Response object from the request. + count(int, optional): Expected count of indices. + index(dict, optional): Expected index data. + indices(list, optional): Expected list of indices. + target_ids(list, optional): List of target index IDs. + item_ids(list, optional): List of item IDs. + """ + expected_response = { + "message": "The server could not verify that you are authorized " + "to access the URL requested. You either supplied the wrong credentials " + "(e.g. a bad password), or your browser doesn't understand how to supply the credentials required.", + "status": 401 + } + assert response.json() == expected_response + + if count is not None: + verify_count(count) + if index is not None: + verify_index(index) + if indices is not None: + verify_indices(indices) + + if target_ids is not None and item_ids is not None: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT is_deleted FROM index WHERE id IN %s;" + cursor.execute(index_query, (tuple(target_ids),)) + index_data = cursor.fetchall() + for data in index_data: + assert data["is_deleted"] == False + + pid_query = "SELECT status FROM pidstore_pid WHERE pid_type = 'recid' AND pid_value IN %s;" + cursor.execute(pid_query, (tuple([str(id) for id in item_ids]),)) + pid_data = cursor.fetchall() + for pid in pid_data: + assert pid["status"] == "R" + + +def verify_forbidden(response, count=None, index=None, indices=None, target_ids=None, item_ids=None): + """Verify that the response indicates a forbidden access with the expected message. + + Args: + response(Response): Response object from the request. + count(int, optional): Expected count of indices. + index(dict, optional): Expected index data. + indices(list, optional): Expected list of indices. + target_ids(list, optional): List of target index IDs. + item_ids(list, optional): List of item IDs. + """ + expected_response = { + "message": "You don't have the permission to access the requested resource. " + "It is either read-protected or not readable by the server.", + "status": 403, + } + assert response.json() == expected_response + + if count is not None: + verify_count(count) + if index is not None: + verify_index(index) + if indices is not None: + verify_indices(indices) + + if target_ids is not None and item_ids is not None: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index_query = "SELECT is_deleted FROM index WHERE id IN %s;" + cursor.execute(index_query, (tuple(target_ids),)) + index_data = cursor.fetchall() + for data in index_data: + assert data["is_deleted"] == False + + pid_query = "SELECT status FROM pidstore_pid WHERE pid_type = 'recid' AND pid_value IN %s;" + cursor.execute(pid_query, (tuple([str(id) for id in item_ids]),)) + pid_data = cursor.fetchall() + for pid in pid_data: + assert pid["status"] == "R" + + +def verify_forbidden_html(response, language, map_groups={}): + """Verify that the response indicates a 'Forbidden' error. + + Args: + response(Response): The response object to check. + language(str): The expected language of the error message. + map_groups(dict, optional): Dictionary containing group mapping information for verification. Defaults to {}. + """ + permisssion_required_messages = { + "en": "Permission required", + "ja": "権限が必要です", + } + soup = BeautifulSoup(response.text, 'html.parser') + h1 = soup.find('h1') + assert h1.text.endswith(permisssion_required_messages[language]) + + if map_groups: + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + if map_groups.get("add"): + role_query = "SELECT id FROM accounts_role WHERE name IN %s;" + cursor.execute(role_query, (tuple(map_groups["add"]),)) + added_roles = cursor.fetchall() + assert len(added_roles) == 0 + elif map_groups.get("delete"): + deleted_roles_ids = json.loads(map_groups["delete"]) + role_query = "SELECT id FROM accounts_role WHERE id IN %s;" + cursor.execute(role_query, (tuple(deleted_roles_ids),)) + deleted_roles = cursor.fetchall() + assert len(deleted_roles) == len(deleted_roles_ids) + + +def verify_internal_server_error(response, count=None, index=None, indices=None): + """Verify that the response indicates an internal server error with the expected message. + + Args: + response(Response): Response object from the request. + count(int, optional): Expected count of indices. + index(dict, optional): Expected index data. + indices(list, optional): Expected list of indices. + """ + expected_response = { + "message": "The server encountered an internal error and was unable " + "to complete your request. Either the server is overloaded or " + "there is an error in the application.", + "status": 500, + } + assert response.json() == expected_response + + if count is not None: + verify_count(count) + if index is not None: + verify_index(index) + if indices is not None: + verify_indices(indices) + + +def verify_internal_server_error_html(response): + """Verify that the response indicates an internal server error in HTML format. + + Args: + response(Response): Response object from the request. + """ + soup = BeautifulSoup(response.text, 'html.parser') + h1 = soup.find('h1') + assert h1.text.endswith("Internal server error") + + +def verify_count(count): + """Verify the count of indices in the database. + + Args: + count(int): Expected count of indices. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cursor: + cursor.execute("SELECT COUNT(*) FROM index;") + actual_count = cursor.fetchone()[0] + assert actual_count == int(count) + + +def verify_index(index): + """Verify the index data in the database. + + Args: + index (dict): Expected index data. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + index = json.loads(str(index)) + replaced_index = {} + for k, v in index.items(): + if isinstance(v, str): + try: + replaced_index[k] = datetime.fromisoformat(v) + except ValueError: + replaced_index[k] = v + else: + replaced_index[k] = v + cursor.execute("SELECT * FROM index WHERE id = %s;", (index["id"],)) + index_data = cursor.fetchone() + assert index_data == replaced_index + + +def verify_indices(indices): + """Verify the list of indices in the database. + + Args: + indices(list): Expected list of indices. + """ + with closing(connect_db()) as conn: + with closing(conn.cursor(cursor_factory=RealDictCursor)) as cursor: + expected_indices = json.loads(str(indices)) + replaced_indices = [] + for index in expected_indices: + replaced_index = {} + for k, v in index.items(): + if isinstance(v, str): + try: + replaced_index[k] = datetime.fromisoformat(v) + except ValueError: + replaced_index[k] = v + else: + replaced_index[k] = v + replaced_indices.append(replaced_index) + cursor.execute("SELECT * FROM index ORDER BY id;") + index_data = cursor.fetchall() + assert index_data == replaced_indices + +def verify_import(response, role=None, code=None, index=None, result={}): + """Verify the import response based on the expected status code and index data. + + Args: + response(Response): Response object from the import request. + role(str, optional): The role of the user performing the import. Defaults to None. + code(int, optional): Expected status code of the response. If None, the function will check the overall import result. Defaults to None. + index(dict, optional): Expected index data for verification. Defaults to None. + result(dict, optional): Dictionary to store the import result for each role. Defaults to {} + + Returns: + Box: A Box object containing the import result for each role if code is provided, otherwise + it will assert the overall import result and print any errors. + """ + if result: + result = json.loads(str(result)) + + if code is not None: + role_result = { + "status": "", + "error": "", + } + try: + if code == 200: + verify_edit_during_import(response, index) + elif code == 401: + verify_unauthorized(response, index=index) + elif code == 403: + verify_forbidden(response, index=index) + + role_result["status"] = "success" + except AssertionError as e: + role_result["status"] = "failed" + role_result["error"] = str(e) + result[role] = role_result + return Box({'import_result': json.dumps(result)}) + else: + is_success = True + for r in result: + if result[r]["status"] == "failed": + is_success = False + print(f"Role: {r}, Error: {result[r]['error']}") + assert is_success == True diff --git a/test/tavern/helper/verify_helper.py b/test/tavern/helper/item_create/verify_helper.py similarity index 94% rename from test/tavern/helper/verify_helper.py rename to test/tavern/helper/item_create/verify_helper.py index 54cdd83baf..e3d0f12d1e 100644 --- a/test/tavern/helper/verify_helper.py +++ b/test/tavern/helper/item_create/verify_helper.py @@ -3,11 +3,13 @@ import difflib from email.header import decode_header import json +import os import pprint from urllib.parse import urlparse, urlunparse from helper.config import INVENIO_WEB_HOST_NAME -from helper.verify_database_helper import connect_db, compare_db_data +from helper.common.connect_helper import connect_db +from helper.common.verify_database_helper import compare_db_data def response_verify_common_response(response, file_name, key): """Verify common response @@ -121,12 +123,13 @@ def response_verify_deposits_items_response(response, file_name, host, excepted= try: assert created_time >= second_before_time and created_time < now_time - assert response.json()['id'] == expect['id'] + response_json = response.json() + assert response_json['id'] == expect['id'] for k in expect['links']: if k == 'bucket': - expect['links'][k].startswith(expect['links'][k]) + assert response_json['links'][k].startswith(expect['links'][k]) else: - assert response.json()['links'][k] == expect['links'][k] + assert response_json['links'][k] == expect['links'][k] except AssertionError as e: assertion_error_handler(e, expect, response.json()) @@ -344,10 +347,7 @@ def response_verify_workflow_records(response, folder_path, activity_id, activit 'initialization': '/api/deposits/items' }, 'files': [], - 'metainfo': expect, - 'weko_link': { - '1': '10' - } + 'metainfo': expect } replace_params = { 'workflow_activity': { @@ -365,7 +365,7 @@ def response_verify_workflow_records(response, folder_path, activity_id, activit 'activity_login_user': 'int', 'activity_update_user': 'int', 'activity_confirm_term_of_use': 'bool', - 'shared_user_id': 'int', + 'shared_user_ids': 'json', 'extra_info': 'json', 'action_order': 'int' } @@ -398,7 +398,7 @@ def response_verify_temp_data_after_change_author_info(response, creator_key): name_identifiers = authors[0]['nameIdentifiers'] for name_identifier in name_identifiers: if name_identifier['nameIdentifierScheme'] == 'WEKO': - assert name_identifier['nameIdentifier'] == '1' + assert name_identifier['nameIdentifier'] == '10' def response_verify_changed_data(response, folder_path, activity_id): """Verify changed data in the database after an activity @@ -457,6 +457,11 @@ def verify_approval_mail(response, approval_type, title_key, data): Returns: None """ + def get_files_sorted_desc(folder_path): + files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))] + files.sort(reverse=True) + return files + data = json.loads(data) if len(title_key.split('.')) == 3: title = data[title_key.split('.')[0]][int(title_key.split('.')[1])][title_key.split('.')[2]] @@ -464,7 +469,8 @@ def verify_approval_mail(response, approval_type, title_key, data): title = data[title_key.split('.')[0]][title_key.split('.')[1]] if approval_type == 'request': - with open('mail/repoadmin', 'r') as f: + mail_files = get_files_sorted_desc('mail/repoadmin/new') + with open(os.path.join('mail/repoadmin/new', mail_files[0]), 'r') as f: request_mail = f.readlines() subject_idx = 0 from_idx = 0 @@ -483,7 +489,8 @@ def verify_approval_mail(response, approval_type, title_key, data): colon_idx = decoded_subject.find(':') assert decoded_subject[colon_idx + 1:].strip() == f"[Approval Request] Please review and approve the item \"{title}\"" elif approval_type == 'complete': - with open('mail/contributor', 'r') as f: + mail_files = get_files_sorted_desc('mail/contributor/new') + with open(os.path.join('mail/contributor/new', mail_files[0]), 'r') as f: request_mail = f.readlines() subject_idx = 0 from_idx = 0 @@ -520,4 +527,4 @@ def assertion_error_handler(e, expect, actual): fromfile='actual', tofile='expected', lineterm='' )) print("差分:\n" + diff) - raise e \ No newline at end of file + raise e diff --git a/test/tavern/helper/item_create_helper.py b/test/tavern/helper/item_create_helper.py deleted file mode 100644 index ec67c3e966..0000000000 --- a/test/tavern/helper/item_create_helper.py +++ /dev/null @@ -1,421 +0,0 @@ -import json -import requests - -from .request_helper import ( - request_create_validate_param, - request_create_save_activity_data_param, - request_create_save_param, - request_create_deposits_items_param, - request_create_deposits_redirect_param, - request_create_deposits_items_index_param, - request_create_action_param -) -from .response_helper import ( - response_save_next_path, - response_save_recid, - response_save_tree_data, - response_save_identifier_grant -) - - -def create_item(response, host, create_info_file, creation_count): - """Create items based on the provided creation information. - - Args: - response: The response object from the initial request. - host: The base URL of the WEKO instance. - create_info_file: Path to the JSON file containing creation information. - creation_count: Number of items to create. - - Raises: - Exception: If any step in the item creation process fails. - """ - # Get the necessary headers from the response - request_headers = response.request.headers - header = { - 'Cookie': request_headers.get('Cookie', ''), - 'X-CSRFToken': request_headers.get('X-CSRFToken', ''), - } - - with open(create_info_file, 'r') as f: - create_info = json.loads(f.read()) - - session = requests.Session() - session.headers.update(header) - - for i in range(creation_count): - # create activity - activity_response = activity_init(host, - session, - create_info['flow_id'], - create_info['itemtype_id'], - create_info['workflow_id']) - - # next path - next_path(host, session, activity_response['next_path']) - - # item_validate - item_validate(host, session, create_info['data_file']) - - # save activity data - save_activity_data( - host, - session, - activity_response['activity_id'], - create_info['data_file'], - create_info['title_key'] - ) - - # iframe model save - iframe_model_save(host, session, create_info['data_file']) - - # deposits item - deposits_response = deposits_items(host, session, create_info['data_file']) - - # deposits redirect - deposits_redirect( - host, - session, - deposits_response['recid'], - create_info['data_file'], - create_info['title_key'] - ) - - # api tree - tree_response = api_tree(host, session, deposits_response['recid']) - - # deposits items recid - deposits_items_recid( - host, - session, - deposits_response['recid'], - str(tree_response['tree_data']) - ) - - # activity 3 - activity_3( - host, - session, - activity_response['activity_id'], - create_info['action_version']['3'] - ) - - # activity 5 - activity_5( - host, - session, - activity_response['activity_id'], - create_info['action_version']['5'] - ) - - # activity detail - activity_detail_response = activity_detail( - host, - session, - activity_response['activity_id'] - ) - - # activity 7 - activity_7( - host, - session, - activity_response['activity_id'], - create_info['action_version']['7'], - activity_detail_response['identifier_grant'] - ) - - # activity 4 - activity_4( - host, - session, - activity_response['activity_id'], - create_info['action_version']['4'] - ) - -def activity_init(host, session, flow_id, itemtype_id, workflow_id): - """Initialize a workflow activity. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - flow_id (str): The ID of the workflow flow. - itemtype_id (str): The ID of the item type. - workflow_id (str): The ID of the workflow. - - Returns: - dict: The response containing the activity ID and next path. - - Raises: - Exception: If the request fails or the response is not as expected. - """ - url = f"{host}/workflow/activity/init" - data = { - 'flow_id': flow_id, - 'itemtype_id': itemtype_id, - 'workflow_id': workflow_id, - } - response = session.post(url, json=data, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to initialize activity: {response.text}") - return response_save_next_path(response) - -def next_path(host, session, path): - """Get the next path in the workflow. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - path (str): The next path to navigate to. - - Raises: - Exception: If the request fails or the response is not as expected. - """ - url = f"{host}{path}" - response = session.get(url, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to get next path: {response.text}") - -def item_validate(host, session, data_file): - """Validate item data. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - data_file (str): Path to the JSON file containing item data to validate. - - Raises: - Exception: If the validation fails or the response is not as expected. - """ - with open(data_file, 'r') as f: - data = f.read() - - params = request_create_validate_param(data) - url = f"{host}/api/items/validate" - response = session.post(url, json=params, verify=False) - - if response.status_code != 200: - raise Exception(f"Validation failed: {response.text}") - -def save_activity_data(host, session, activity_id, data_file, title_key): - """Save activity data. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to save data for. - data_file (str): Path to the JSON file containing activity data to save. - title_key (str): The key for the title in the data. - - Raises: - Exception: If the save operation fails or the response is not as expected. - """ - with open(data_file, 'r') as f: - data = f.read() - - params = request_create_save_activity_data_param(activity_id, data, title_key) - url = f"{host}/workflow/save_activity_data" - response = session.post(url, json=params, verify=False) - - if response.status_code != 200: - raise Exception(f"Failed to save activity data: {response.text}") - -def iframe_model_save(host, session, data_file): - """Save item data in iframe model format. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - data_file (str): Path to the JSON file containing item data to save. - - Raises: - Exception: If the save operation fails or the response is not as expected. - """ - with open(data_file, 'r') as f: - data = f.read() - - params = request_create_save_param(data) - url = f"{host}/items/iframe/model/save" - response = session.post(url, json=params, verify=False) - - if response.status_code != 200: - raise Exception(f"Failed to save item: {response.text}") - -def deposits_items(host, session, data_file): - """Deposit items based on the provided data file. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - data_file (str): Path to the JSON file containing item data to deposit. - - Returns: - dict: The response containing the recid of the deposited item. - - Raises: - Exception: If the deposit operation fails or the response is not as expected. - """ - with open(data_file, 'r') as f: - data = f.read() - - params = request_create_deposits_items_param(data) - url = f"{host}/api/deposits/items" - response = session.post(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to deposit item: {response.text}") - - return response_save_recid(response) - -def deposits_redirect(host, session, recid, data_file, title_key): - """Redirect a deposit based on the provided recid and data file. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - recid (str): The recid of the deposit to redirect. - data_file (str): Path to the JSON file containing data for redirection. - title_key (str): The key for the title in the data. - - Raises: - Exception: If the redirection fails or the response is not as expected. - """ - with open(data_file, 'r') as f: - data = f.read() - - params = request_create_deposits_redirect_param(data, title_key) - url = f"{host}/api/deposits/redirect/{recid}" - response = session.put(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to redirect deposit: {response.text}") - -def api_tree(host, session, recid): - """Get the tree data for a specific recid. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - recid (str): The recid of the item to get tree data for. - - Returns: - dict: The response containing the tree data. - - Raises: - Exception: If the request fails or the response is not as expected. - """ - url = f"{host}/api/tree/{recid}" - response = session.get(url, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to get tree: {response.text}") - - return response_save_tree_data(response) - -def deposits_items_recid(host, session, recid, tree_data): - """Update deposits items with the provided recid and tree data. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - recid (str): The recid of the item to update. - tree_data (str): The tree data to update the item with. - - Raises: - Exception: If the update operation fails or the response is not as expected. - """ - params = request_create_deposits_items_index_param(tree_data) - url = f"{host}/api/deposits/items/{recid}" - response = session.put(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to get deposits items: {response.text}") - -def activity_3(host, session, activity_id, version): - """Perform activity action 3. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to perform action on. - version (str): The version of the action to perform. - - Raises: - Exception: If the action fails or the response is not as expected. - """ - params = request_create_action_param(version) - url = f"{host}/workflow/activity/action/{activity_id}/3" - response = session.post(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to perform activity action: {response.text}") - -def activity_5(host, session, activity_id, version): - """Perform activity action 5. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to perform action on. - version (str): The version of the action to perform. - - Raises: - Exception: If the action fails or the response is not as expected. - """ - params = request_create_action_param(version, link_data=[]) - url = f"{host}/workflow/activity/action/{activity_id}/5" - response = session.post(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to perform activity action: {response.text}") - -def activity_detail(host, session, activity_id): - """Get the details of a specific activity. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to get details for. - - Returns: - dict: The response containing the activity details, including identifier grant. - - Raises: - Exception: If the request fails or the response is not as expected. - """ - url = f"{host}/workflow/activity/detail/{activity_id}?page=1&size=20" - response = session.get(url, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to get activity detail: {response.text}") - - return response_save_identifier_grant(response) - -def activity_7(host, session, activity_id, version, identifier): - """Perform activity action 7. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to perform action on. - version (str): The version of the action to perform. - identifier (str): The identifier to use in the action. - - Raises: - Exception: If the action fails or the response is not as expected. - """ - params = request_create_action_param(version, identifier=identifier) - url = f"{host}/workflow/activity/action/{activity_id}/7" - response = session.post(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to perform activity action: {response.text}") - -def activity_4(host, session, activity_id, version): - """Perform activity action 4. - - Args: - host (str): The base URL of the WEKO instance. - session (requests.Session): The session object for making requests. - activity_id (str): The ID of the activity to perform action on. - version (str): The version of the action to perform. - - Raises: - Exception: If the action fails or the response is not as expected. - """ - params = request_create_action_param(version, community='') - url = f"{host}/workflow/activity/action/{activity_id}/4" - response = session.post(url, json=params, verify=False) - if response.status_code != 200: - raise Exception(f"Failed to perform activity action: {response.text}") diff --git a/test/tavern/helper/item_type_mapping/request_helper.py b/test/tavern/helper/item_type_mapping/request_helper.py new file mode 100644 index 0000000000..ff09fc7105 --- /dev/null +++ b/test/tavern/helper/item_type_mapping/request_helper.py @@ -0,0 +1,10 @@ +import json + +def generate_request_body(file_path): + """Generate a request body by reading a JSON file. + + Args: + file_path(str): The path to the JSON file. + """ + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) diff --git a/test/tavern/helper/item_type_mapping/response_helper.py b/test/tavern/helper/item_type_mapping/response_helper.py new file mode 100644 index 0000000000..d6ef7c681e --- /dev/null +++ b/test/tavern/helper/item_type_mapping/response_helper.py @@ -0,0 +1,74 @@ +import random +from contextlib import closing + +from box import Box +from helper.common.connect_helper import connect_db + +def delete_itemtypes(_): + """Delete all item types and their related mappings from the database. + + Args: + _(Response): Placeholder parameter for compatibility with the caller. + """ + delete_query = [ + "TRUNCATE TABLE item_type_name CASCADE;", + "TRUNCATE TABLE item_type_mapping;", + "TRUNCATE TABLE item_type_mapping_version;", + "TRUNCATE TABLE item_type_version;" + ] + + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cursor: + for query in delete_query: + cursor.execute(query) + conn.commit() + + +def delete_xsd(_): + """Set the xsd field to NULL for a random record in the oaiserver_schema table. + + Args: + _(Response): Placeholder parameter for compatibility with the caller. + + Returns: + Box: A Box object containing the schema_name of the updated record. + + Raises: + Exception: If no records are found in the oaiserver_schema table. + """ + # Set the xsd field to NULL for a random record in the oaiserver_schema table + select_query = "SELECT id, schema_name FROM oaiserver_schema;" + update_query = "UPDATE oaiserver_schema SET xsd = '{}' WHERE id = %s;" + + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cursor: + cursor.execute(select_query) + records = cursor.fetchall() + if records: + random_record = random.choice(records) + cursor.execute(update_query, (random_record[0],)) + conn.commit() + return Box({"schema_name": random_record[1]}) + else: + raise Exception("No records found in oaiserver_schema table.") + + +def get_table_rows_count(_, table_names): + """Get the row count for the specified tables. + + Args: + _(Response): Placeholder parameter for compatibility with the caller. + table_names(list): A list of table names to retrieve row counts for. + + Returns: + Box: A Box object containing the row count for each specified table, + structured as {"row_count": {table_name: count}}. + """ + count_dict = {} + with closing(connect_db()) as conn: + with closing(conn.cursor()) as cursor: + for table_name in table_names: + cursor.execute(f"SELECT COUNT(*) FROM {table_name}") + count = cursor.fetchone()[0] + count_dict[table_name] = count + return Box({"row_count": count_dict}) diff --git a/test/tavern/helper/item_type_mapping/verify_helper.py b/test/tavern/helper/item_type_mapping/verify_helper.py new file mode 100644 index 0000000000..1376ebebd8 --- /dev/null +++ b/test/tavern/helper/item_type_mapping/verify_helper.py @@ -0,0 +1,653 @@ +import copy +import json +from bs4 import BeautifulSoup +from contextlib import closing +from urllib.parse import urlencode +import xmltodict + +from helper.common.connect_helper import connect_db + +def verify_contains(response, expected_substring): + """Verify that the response contains the expected substring. + + Args: + response(Response): The response object to check. + expected_substring(str): The substring expected to be found in the response content. + + Raises: + AssertionError: If the expected substring is not found in the response content. + """ + content = response.text + assert expected_substring in content + + +def verify_selected_value(response, select_id, expected_value, is_sysadmin): + """Verify that the selected value in a dropdown matches the expected value. + + Args: + response(Response): The response object containing HTML content. + select_id(str): The id attribute of the select element to check. + expected_value(str): The expected selected value. + is_sysadmin(bool): Boolean indicating if the user is a system administrator. + + Raises: + AssertionError: If the select element or options do not meet the expected conditions. + """ + soup = BeautifulSoup(response.text, 'html.parser') + + # Find the select element by id + select_element = soup.find('select', id=select_id) + if not select_element: + raise AssertionError(f"Select element with id '{select_id}' not found.") + + # Find all option elements within the select + options = select_element.find_all('option') + if not options: + raise AssertionError(f"No options found in select element with id '{select_id}'.") + + # Check if the expected value is selected and if other options are not selected + for option in options: + option_value = option.get('value') + if option_value == expected_value: + assert option.has_attr('selected') + else: + assert not option.has_attr('selected') + + if not is_sysadmin: + # Verify that all radio buttons and parent lists are disabled for non-sysadmin users + radio_parent_lists = soup.find_all(name='radio_parent_list') + assert all(radio.has_attr('disabled') for radio in radio_parent_lists) + parent_lists = soup.find_all(name='parent_list') + assert all(parent.has_attr('disabled') for parent in parent_lists) + + +def verify_select_invalid_mapping(response): + """Verify that the response contains a list with specific classes in its
  • elements. + + Args: + response(Response): The response object containing HTML content. + + Raises: + AssertionError: If the