From 54b2f03db3862d1a928e9e5c8f11f3c1906b72d2 Mon Sep 17 00:00:00 2001 From: mohithg Date: Sun, 17 Oct 2021 11:48:27 +0530 Subject: [PATCH 1/4] Implement fds pull --- fds/cli.py | 13 +++++++++++++ fds/domain/commands.py | 1 + fds/run.py | 4 ++++ fds/services/fds_service.py | 17 +++++++++++++++++ fds/services/git_service.py | 17 +++++++++++++++++ 5 files changed, 52 insertions(+) diff --git a/fds/cli.py b/fds/cli.py index 12edf864..2aac6486 100644 --- a/fds/cli.py +++ b/fds/cli.py @@ -45,6 +45,19 @@ help="A git branch that you want to push to (accepts refspec), defaults to current branch", nargs='?' ) + +# pull +parser_pull = command_subparser.add_parser( + 'pull', + help='pull commits from remote git and dvc repository' +) +parser_pull.add_argument('-gr', '--git-remote', help="git remote name, default 'origin'", default="origin") +parser_pull.add_argument('-dr', '--dvc-remote', help="dvc remote name, default 'origin'", default="origin") +parser_pull.add_argument( + 'branch', + help="A git branch that you want to pull from to (accepts refspec), defaults to current branch", + nargs='?' +) # save parser_save = command_subparser.add_parser( 'save', diff --git a/fds/domain/commands.py b/fds/domain/commands.py index eba9cce5..74492922 100644 --- a/fds/domain/commands.py +++ b/fds/domain/commands.py @@ -8,6 +8,7 @@ class Commands(Enum): ADD = "add" COMMIT = "commit" PUSH = "push" + PULL = "pull" SAVE = "save" CLONE = "clone" VERSION = "version" diff --git a/fds/run.py b/fds/run.py index dd99422f..180a7b14 100644 --- a/fds/run.py +++ b/fds/run.py @@ -221,6 +221,10 @@ def execute(self): # Run push command stuff self.service.push(arguments["git_remote"], arguments["dvc_remote"], arguments["branch"]) return 0 + elif arguments["command"] == Commands.PULL.value: + # Run pull command stuff + self.service.pull(arguments["git_remote"], arguments["dvc_remote"], arguments["branch"]) + return 0 elif arguments["command"] == Commands.SAVE.value: # Run save command stuff self.service.save(arguments["message"], arguments["git_remote"], arguments["dvc_remote"]) diff --git a/fds/services/fds_service.py b/fds/services/fds_service.py index ddd47e78..6cbbbf24 100644 --- a/fds/services/fds_service.py +++ b/fds/services/fds_service.py @@ -140,6 +140,23 @@ def push(self, git_remote: str, dvc_remote: str, ref: str = None): self.printer.error(str(e)) raise Exception("DVC push failed to execute") + def pull(self, git_remote: str, dvc_remote: str, ref: str = None): + """ + fds pull + """ + try: + git_url = self.git_service.pull(remote=git_remote, ref=ref) + self.printer.success("Successfully pulled from Git remote") + except Exception as e: + self.printer.error(str(e)) + raise Exception("Git pull failed to execute") + try: + self.dvc_service.pull(git_url, remote_name=dvc_remote) + self.printer.warn("Successfully pulled to DVC remote") + except Exception as e: + self.printer.error(str(e)) + raise Exception("DVC pull failed to execute") + def save(self, message: str, git_remote: str, dvc_remote: str): self.add(".") self.commit(message) diff --git a/fds/services/git_service.py b/fds/services/git_service.py index 9eb154e5..da81324f 100644 --- a/fds/services/git_service.py +++ b/fds/services/git_service.py @@ -88,6 +88,23 @@ def push(remote: str, ref: str) -> Any: push_cmd.append(curr_branch) execute_command(push_cmd, capture_output=False) + @staticmethod + def pull(remote: str, ref: Optional[str]) -> Any: + pull_cmd = ["git", "pull"] + pull_cmd.append(remote) + if ref: + pull_cmd.append(ref) + else: + check_curr_branch = execute_command(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + curr_branch = convert_bytes_to_string(check_curr_branch.stdout).rstrip('\n') + if curr_branch == '': + raise Exception("No git branch found to pull from") + pull_cmd.append(curr_branch) + execute_command(pull_cmd, capture_output=False) + git_url_cmd = execute_command(["git", "config", "--get", "remote.origin.url"]) + git_url = convert_bytes_to_string(git_url_cmd.stdout).rstrip('\n') + return git_url + def clone(self, url: str, folder_name: Optional[str]) -> Any: if folder_name is None or folder_name == "": folder_name = get_git_repo_name_from_url(url) From 2d24203a094b24ca8d3117a6a408e0ce93bd09e6 Mon Sep 17 00:00:00 2001 From: mohithg Date: Sun, 17 Oct 2021 14:43:19 +0530 Subject: [PATCH 2/4] Add tests --- tests/it/test_dvc.py | 12 ++++++++++++ tests/it/test_fds.py | 8 ++++++++ tests/it/test_git.py | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/tests/it/test_dvc.py b/tests/it/test_dvc.py index 8fc8173d..4b4d10eb 100644 --- a/tests/it/test_dvc.py +++ b/tests/it/test_dvc.py @@ -188,3 +188,15 @@ def test_get_repo_path(self): self.create_dummy_folder("test_dvc") path = self.dvc_service.get_repo_path() assert path == self.repo_path + + def test_pull_default_remote(self): + folder_name = self.git_service.clone(self.get_remote_url_for_test(), None) + os.chdir(folder_name) + self.dvc_service.pull(self.get_remote_url_for_test(), None) + assert does_file_exist(f"{self.repo_path}/hello-world/data") + + def test_pull_with_remote(self): + folder_name = self.git_service.clone(self.get_remote_url_for_test(), None) + os.chdir(folder_name) + self.dvc_service.pull(self.get_remote_url_for_test(), "origin") + assert does_file_exist(f"{self.repo_path}/hello-world/data") diff --git a/tests/it/test_fds.py b/tests/it/test_fds.py index bdc9e78f..62fb5f5c 100644 --- a/tests/it/test_fds.py +++ b/tests/it/test_fds.py @@ -135,3 +135,11 @@ def test_clone_given_remote(self): assert does_file_exist(f"{self.repo_path}/start") # Checking dvc pull of storage remote assert does_file_exist(f"{self.repo_path}/start/data/data.xml") + + def test_pull(self): + self.fds_service.clone(self.get_remote_url_for_test(), None, None) + # Check git clone + assert does_file_exist(f"{self.repo_path}/hello-world") + # Checking dvc pull + assert does_file_exist(f"{self.repo_path}/hello-world/data") + self.fds_service.pull("origin", "Origin", None) diff --git a/tests/it/test_git.py b/tests/it/test_git.py index 4fb037b0..1cad02de 100644 --- a/tests/it/test_git.py +++ b/tests/it/test_git.py @@ -1,3 +1,5 @@ +import os + from fds.utils import does_file_exist, execute_command, convert_bytes_to_string from tests.it.helpers import IntegrationTestCase @@ -101,3 +103,13 @@ def test_get_repo_path(self): self.create_dummy_folder("test_git") path = self.git_service.get_repo_path() assert path == self.repo_path + + def test_pull(self): + self.git_service.clone(self.get_remote_url_for_test(), None) + os.chdir(f"{self.repo_path}/hello-world") + self.git_service.pull("origin", None) + + def test_pull_branch(self): + self.git_service.clone(self.get_remote_url_for_test(), None) + os.chdir(f"{self.repo_path}/hello-world") + self.git_service.pull("origin", "main") From 62411e3b71444fb9f738782f29cf355d06d59c5a Mon Sep 17 00:00:00 2001 From: mohithg Date: Sun, 17 Oct 2021 15:28:13 +0530 Subject: [PATCH 3/4] Fix test --- tests/it/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/it/test_git.py b/tests/it/test_git.py index 1cad02de..23bed586 100644 --- a/tests/it/test_git.py +++ b/tests/it/test_git.py @@ -112,4 +112,4 @@ def test_pull(self): def test_pull_branch(self): self.git_service.clone(self.get_remote_url_for_test(), None) os.chdir(f"{self.repo_path}/hello-world") - self.git_service.pull("origin", "main") + self.git_service.pull("origin", "master") From 2e7f07cf676bfe3556078de4a55b40d46c82ee19 Mon Sep 17 00:00:00 2001 From: mohithg Date: Sun, 17 Oct 2021 16:24:42 +0530 Subject: [PATCH 4/4] Fix issues with pip install --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3de9fea7..ea8b4537 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install 'pygit2==1.6' if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi pip install 'dvc==2.3.0'