Add flask routes to launch and destroy cloudvolume and neuroglancer containers

.env

@@ -1,4 +1,10 @@
+HOSTURL=PNI-192QMG3Y2
 CVDATA=./data
 CONFIGPROXY_AUTH_TOKEN='31507a9ddf3e41cf86b58ffede2db68326657437704461ae2c1a4018d55e18f0'
 TZ=America/New_York
 PYTHONPATH=/opt/libraries
+
+NB_UID=1001
+NB_GID=1001		
+
+

.gitignore

@@ -1,12 +1,11 @@
-
 # Created by https://www.gitignore.io/api/python
 # Edit at https://www.gitignore.io/?templates=python
 
 ### Python ###
 # Byte-compiled / optimized / DLL files
-__pycache__/
 *.py[cod]
 *$py.class
+**/__pycache__/
 
 # C extensions
 *.so
@@ -32,6 +31,7 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -117,3 +117,10 @@ dmypy.json
 
 
 data/.*
+**/*pkl
+**/.ipynb_checkpoints
+**/.cache
+**/.ipython
+**/.jupyter
+**/.bash_history
+**/.local

apache/sites/default-vhost.conf

@@ -23,12 +23,19 @@ RequestHeader set X-Remote-User 'testuser'
   ProxyPass "/vols" "http://confproxy:8000" flushpackets=on
   ProxyPassReverse "/vols" "http://confproxy:8000"
 
-
   <Location "/nglancer">
        ## neuroglancer
       RequestHeader set Origin "http://confproxy:8888"
-      ProxyPass  "http://confproxy:8000/viewer0/" flushpackets=on
-      ProxyPassReverse  "http://confproxy:8000/viewer0/"
+      ProxyPass  "http://confproxy:8000/viewers" flushpackets=on
+      ProxyPassReverse  "http://confproxy:8000/viewers"
+
+  </Location>
+
+  <Location "/cv">
+       ## cloudvolume
+      RequestHeader set Origin "http://confproxy:8888"
+      ProxyPass  "http://confproxy:8000/cloudvols" flushpackets=on
+      ProxyPassReverse  "http://confproxy:8000/cloudvols"
 
   </Location>
 

cloudvolume/volshim.py

@@ -7,18 +7,17 @@
 import numpy as np
 import types
 from time import sleep
-
-def start_server():
+# import progproxy as pp
 
 
+def start_server():
 
     ## add some error checking
     if not os.path.isfile('/mnt/data/info'):
         logging.info('no valid volume found, using test data')
         arr = np.random.random_integers(0, high=255, size=(128,128, 128))
         arr = np.asarray(arr, dtype=np.uint8)
         vol = CloudVolume.from_numpy(arr, max_mip=1)
-
     else:
         logging.info('using mounted dataset')
         vol = CloudVolume('file:///mnt/data',parallel=2,cache=True)
@@ -30,9 +29,8 @@ def start_server():
     vol.viewer = funcType(localviewer, vol)
 
     logging.info('starting cloudvolume service')
-    vol.viewer()
-
 
+    vol.viewer(port=1337)
 
 
 ## replacement version of the 'view' function for the object `CloudVolumePrecomputed`
@@ -44,5 +42,11 @@ def localviewer(self, port=1337):
 
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
+    # logging.basicConfig(level=logging.DEBUG)
+    # logging.info(os.listdir('/mnt/data/'))
+    # logging.info(pp)
+    # proxy_h = pp.progproxy(target_hname="confproxy")
+    # proxy_h.getroutes()
+    # register_cloudvol_confproxy()
+    # logging.info("made it here")
     start_server()

data/.bash_history

@@ -0,0 +1 @@
+history

data/MWE.ipynb

@@ -0,0 +1,225 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2020-01-21 16:50:38,802 - progproxy - DEBUG - Programable Proxy Object Created\n",
+      "2020-01-21 16:50:38,803 - progproxy - DEBUG - Target set to http://confproxy:8001\n",
+      "2020-01-21 16:50:38,804 - progproxy - DEBUG - header set to {'Authorization': \"token '31507a9ddf3e41cf86b58ffede2db68326657437704461ae2c1a4018d55e18f0'\"}\n",
+      "2020-01-21 16:50:38,814 - progproxy - DEBUG - adding path cloudv1 pointing to http://cloudv1:1337\n",
+      "2020-01-21 16:50:38,820 - progproxy - DEBUG - path added successfully\n",
+      "2020-01-21 16:50:40,352 - progproxy - DEBUG - adding path cloudv2 pointing to http://cloudv2:1337\n",
+      "2020-01-21 16:50:40,357 - progproxy - DEBUG - path added successfully\n"
+     ]
+    }
+   ],
+   "source": [
+    "import docker\n",
+    "import progproxy as pp\n",
+    "client = docker.DockerClient(base_url='unix://var/run/docker.sock')\n",
+    "\n",
+    "\n",
+    "environment = {\n",
+    "    'PYTHONPATH':'/opt/libraries',\n",
+    "    'CONFIGPROXY_AUTH_TOKEN':\"'31507a9ddf3e41cf86b58ffede2db68326657437704461ae2c1a4018d55e18f0'\"\n",
+    "}\n",
+    "\n",
+    "# Set up the first cloudvolume container\n",
+    "volumes1 = {\n",
+    "    '/jukebox/LightSheetData/atlas/neuroglancer/atlas/princetonmouse':{\n",
+    "        'bind':'/mnt/data',\n",
+    "        'mode':'ro'\n",
+    "    },\n",
+    "    '/home/ahoag/Git/nglancer-frame/libraries':{\n",
+    "        'bind':'/opt/libraries'\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "container1 = client.containers.run('cloudv_test',\n",
+    "                                  volumes=volumes1,\n",
+    "                                  environment=environment,\n",
+    "                                  network='nglancer',\n",
+    "                                  name='cloudv1',\n",
+    "                                  detach=True)\n",
+    "# Register the cloudvolume with the reverse proxy\n",
+    "proxy_h = pp.progproxy(target_hname='confproxy')\n",
+    "proxy_h.addroute(proxypath='cloudv1',proxytarget=\"http://cloudv1:1337\")\n",
+    "\n",
+    "\n",
+    "# Set up the second cloudvolume container\n",
+    "volumes2 = {\n",
+    "    '/home/ahoag/ngdemo/demo_bucket/grin_lens/oostland_m27':{\n",
+    "        'bind':'/mnt/data',\n",
+    "        'mode':'ro'\n",
+    "    },\n",
+    "    '/home/ahoag/Git/nglancer-frame/libraries':{\n",
+    "        'bind':'/opt/libraries'\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "\n",
+    "container2 = client.containers.run('cloudv_test',\n",
+    "                                  volumes=volumes2,\n",
+    "                                  environment=environment,\n",
+    "                                  network='nglancer',\n",
+    "                                  name='cloudv2',\n",
+    "                                  detach=True)\n",
+    "# Register the cloudvolume with the reverse proxy\n",
+    "proxy_h.addroute(proxypath='cloudv2',proxytarget=\"http://cloudv2:1337\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"data_type\": \"uint16\",\n",
+      "  \"mesh\": \"mesh_mip_0_err_40\",\n",
+      "  \"num_channels\": 1,\n",
+      "  \"scales\": [\n",
+      "    {\n",
+      "      \"chunk_sizes\": [\n",
+      "        [\n",
+      "          1024,\n",
+      "          1024,\n",
+      "          1\n",
+      "        ]\n",
+      "      ],\n",
+      "      \"encoding\": \"raw\",\n",
+      "      \"key\": \"40000_40000_40000\",\n",
+      "      \"resolution\": [\n",
+      "        40000,\n",
+      "        40000,\n",
+      "        40000\n",
+      "      ],\n",
+      "      \"size\": [\n",
+      "        352,\n",
+      "        640,\n",
+      "        540\n",
+      "      ],\n",
+      "      \"voxel_offset\": [\n",
+      "        0,\n",
+      "        0,\n",
+      "        1\n",
+      "      ]\n",
+      "    }\n",
+      "  ],\n",
+      "  \"type\": \"segmentation\"\n",
+      "}\n",
+      "\n",
+      "{\n",
+      "  \"data_type\": \"uint16\",\n",
+      "  \"num_channels\": 1,\n",
+      "  \"scales\": [\n",
+      "    {\n",
+      "      \"chunk_sizes\": [\n",
+      "        [\n",
+      "          1024,\n",
+      "          1024,\n",
+      "          1\n",
+      "        ]\n",
+      "      ],\n",
+      "      \"encoding\": \"raw\",\n",
+      "      \"key\": \"20000_20000_20000\",\n",
+      "      \"resolution\": [\n",
+      "        20000,\n",
+      "        20000,\n",
+      "        20000\n",
+      "      ],\n",
+      "      \"size\": [\n",
+      "        352,\n",
+      "        640,\n",
+      "        540\n",
+      "      ],\n",
+      "      \"voxel_offset\": [\n",
+      "        0,\n",
+      "        0,\n",
+      "        0\n",
+      "      ]\n",
+      "    }\n",
+      "  ],\n",
+      "  \"type\": \"image\"\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "header = {'Authorization': \"token '31507a9ddf3e41cf86b58ffede2db68326657437704461ae2c1a4018d55e18f0'\"}\n",
+    "response1 = requests.get('http://confproxy:8000/cloudv1/info',headers=header)\n",
+    "print(response1.text)\n",
+    "print()\n",
+    "response2 = requests.get('http://confproxy:8000/cloudv2/info',headers=header)\n",
+    "print(response2.text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"/viewer0\":{\"target\":\"http://nglancer:8080/\",\"last_activity\":\"2020-01-21T21:45:22.666Z\"},\"/testvol\":{\"target\":\"http://host.docker.internal:1341\",\"last_activity\":\"2020-01-17T22:15:22.362Z\"},\"/vols/testvol\":{\"target\":\"http://localhost:1340\",\"last_activity\":\"2020-01-17T21:07:14.835Z\"},\"/testvol2\":{\"target\":\"http://mytesthost:1341\",\"last_activity\":\"2020-01-17T21:51:36.661Z\"},\"/testvol3\":{\"target\":\"http://mytestname:1341\",\"last_activity\":\"2020-01-21T21:04:00.773Z\"},\"/cloudv1\":{\"target\":\"http://cloudv1:1337\",\"last_activity\":\"2020-01-21T21:52:00.174Z\"},\"/cloudv2\":{\"target\":\"http://cloudv2:1337\",\"last_activity\":\"2020-01-21T21:52:00.217Z\"}}'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response=requests.get('http://confproxy:8001/api/routes',headers=header)\n",
+    "response.text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}