Implement fds pull

.github/workflows/test.yml

@@ -27,6 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install 'pygit2==1.6'
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
         pip install 'dvc==2.3.0'

fds/cli.py

@@ -45,6 +45,19 @@
     help="A git branch that you want to push to (accepts refspec), defaults to current branch",
     nargs='?'
 )
+
+# pull
+parser_pull = command_subparser.add_parser(
+    'pull',
+    help='pull commits from remote git and dvc repository'
+)
+parser_pull.add_argument('-gr', '--git-remote', help="git remote name, default 'origin'", default="origin")
+parser_pull.add_argument('-dr', '--dvc-remote', help="dvc remote name, default 'origin'", default="origin")
+parser_pull.add_argument(
+    'branch',
+    help="A git branch that you want to pull from to (accepts refspec), defaults to current branch",
+    nargs='?'
+)
 # save
 parser_save = command_subparser.add_parser(
     'save',

fds/domain/commands.py

@@ -8,6 +8,7 @@ class Commands(Enum):
     ADD = "add"
     COMMIT = "commit"
     PUSH = "push"
+    PULL = "pull"
     SAVE = "save"
     CLONE = "clone"
     VERSION = "version"

fds/run.py

@@ -221,6 +221,10 @@ def execute(self):
             # Run push command stuff
             self.service.push(arguments["git_remote"], arguments["dvc_remote"], arguments["branch"])
             return 0
+        elif arguments["command"] == Commands.PULL.value:
+            # Run pull command stuff
+            self.service.pull(arguments["git_remote"], arguments["dvc_remote"], arguments["branch"])
+            return 0
         elif arguments["command"] == Commands.SAVE.value:
             # Run save command stuff
             self.service.save(arguments["message"], arguments["git_remote"], arguments["dvc_remote"])

fds/services/fds_service.py

@@ -140,6 +140,23 @@ def push(self, git_remote: str, dvc_remote: str, ref: str = None):
             self.printer.error(str(e))
             raise Exception("DVC push failed to execute")
 
+    def pull(self, git_remote: str, dvc_remote: str, ref: str = None):
+        """
+        fds pull
+        """
+        try:
+            git_url = self.git_service.pull(remote=git_remote, ref=ref)
+            self.printer.success("Successfully pulled from Git remote")
+        except Exception as e:
+            self.printer.error(str(e))
+            raise Exception("Git pull failed to execute")
+        try:
+            self.dvc_service.pull(git_url, remote_name=dvc_remote)
+            self.printer.warn("Successfully pulled to DVC remote")
+        except Exception as e:
+            self.printer.error(str(e))
+            raise Exception("DVC pull failed to execute")
+
     def save(self, message: str, git_remote: str, dvc_remote: str):
         self.add(".")
         self.commit(message)

fds/services/git_service.py

@@ -88,6 +88,23 @@ def push(remote: str, ref: str) -> Any:
                 push_cmd.append(curr_branch)
         execute_command(push_cmd, capture_output=False)
 
+    @staticmethod
+    def pull(remote: str, ref: Optional[str]) -> Any:
+        pull_cmd = ["git", "pull"]
+        pull_cmd.append(remote)
+        if ref:
+            pull_cmd.append(ref)
+        else:
+            check_curr_branch = execute_command(["git", "rev-parse", "--abbrev-ref", "HEAD"])
+            curr_branch = convert_bytes_to_string(check_curr_branch.stdout).rstrip('\n')
+            if curr_branch == '':
+                raise Exception("No git branch found to pull from")
+            pull_cmd.append(curr_branch)
+        execute_command(pull_cmd, capture_output=False)
+        git_url_cmd = execute_command(["git", "config", "--get", "remote.origin.url"])
+        git_url = convert_bytes_to_string(git_url_cmd.stdout).rstrip('\n')
+        return git_url
+
     def clone(self, url: str, folder_name: Optional[str]) -> Any:
         if folder_name is None or folder_name == "":
             folder_name = get_git_repo_name_from_url(url)

tests/it/test_dvc.py

@@ -188,3 +188,15 @@ def test_get_repo_path(self):
         self.create_dummy_folder("test_dvc")
         path = self.dvc_service.get_repo_path()
         assert path == self.repo_path
+
+    def test_pull_default_remote(self):
+        folder_name = self.git_service.clone(self.get_remote_url_for_test(), None)
+        os.chdir(folder_name)
+        self.dvc_service.pull(self.get_remote_url_for_test(), None)
+        assert does_file_exist(f"{self.repo_path}/hello-world/data")
+
+    def test_pull_with_remote(self):
+        folder_name = self.git_service.clone(self.get_remote_url_for_test(), None)
+        os.chdir(folder_name)
+        self.dvc_service.pull(self.get_remote_url_for_test(), "origin")
+        assert does_file_exist(f"{self.repo_path}/hello-world/data")

tests/it/test_fds.py

@@ -135,3 +135,11 @@ def test_clone_given_remote(self):
         assert does_file_exist(f"{self.repo_path}/start")
         # Checking dvc pull of storage remote
         assert does_file_exist(f"{self.repo_path}/start/data/data.xml")
+
+    def test_pull(self):
+        self.fds_service.clone(self.get_remote_url_for_test(), None, None)
+        # Check git clone
+        assert does_file_exist(f"{self.repo_path}/hello-world")
+        # Checking dvc pull
+        assert does_file_exist(f"{self.repo_path}/hello-world/data")
+        self.fds_service.pull("origin", "Origin", None)

tests/it/test_git.py

@@ -1,3 +1,5 @@
+import os
+
 from fds.utils import does_file_exist, execute_command, convert_bytes_to_string
 from tests.it.helpers import IntegrationTestCase
 
@@ -101,3 +103,13 @@ def test_get_repo_path(self):
         self.create_dummy_folder("test_git")
         path = self.git_service.get_repo_path()
         assert path == self.repo_path
+
+    def test_pull(self):
+        self.git_service.clone(self.get_remote_url_for_test(), None)
+        os.chdir(f"{self.repo_path}/hello-world")
+        self.git_service.pull("origin", None)
+
+    def test_pull_branch(self):
+        self.git_service.clone(self.get_remote_url_for_test(), None)
+        os.chdir(f"{self.repo_path}/hello-world")
+        self.git_service.pull("origin", "master")