From 8f7c7932b0b859d303141d92126d5edbbdb98771 Mon Sep 17 00:00:00 2001 From: mobab-th <4n6linux@gmail.com> Date: Tue, 11 Feb 2025 13:57:36 +0100 Subject: [PATCH 1/2] Adaptation for Taskbridge These changes enable the use of Taskbridge (https://github.com/hilderonny/taskbridge) for IPED in a Docker environment. --- Dockerfile.processor | 23 +++++++- entrypoint.sh | 137 +++++++++++++++++++++---------------------- 2 files changed, 87 insertions(+), 73 deletions(-) diff --git a/Dockerfile.processor b/Dockerfile.processor index 08cbb03..adf8ee3 100644 --- a/Dockerfile.processor +++ b/Dockerfile.processor @@ -1,6 +1,6 @@ -FROM ipeddocker/iped:dependencies +FROM localhost/iped:dependencies -ARG SNAPSHOT=true IPED_RELEASE_VERSION=4.2 SNAPSHOT_WORKFLOW_ID=12279329706 PKGTMPDIR=/tmp/pkgs +ARG SNAPSHOT=false IPED_RELEASE_VERSION=4.2.0 SNAPSHOT_WORKFLOW_ID=12279329706 PKGTMPDIR=/tmp/pkgs RUN --mount=type=secret,id=ACTION_GH_TOKEN export ACTION_GH_TOKEN=$(cat /run/secrets/ACTION_GH_TOKEN) \ @@ -28,6 +28,7 @@ RUN --mount=type=secret,id=ACTION_GH_TOKEN export ACTION_GH_TOKEN=$(cat /run/sec curl -L https://github.com/sepinf-inc/IPED/releases/download/$IPED_RELEASE_VERSION/IPED-${IPED_RELEASE_VERSION}_plus_java_plugins.zip --output ${PKGTMPDIR}/iped.zip && \ unzip ${PKGTMPDIR}/iped.zip ;\ fi \ + && unzip ${PKGTMPDIR}/iped.zip \ && echo "Creating IPED simbolic link..." \ && ls | grep "iped-" | xargs -i sh -c 'ln -s "{}" iped' \ && echo "#####################################" \ @@ -91,6 +92,24 @@ RUN --mount=type=secret,id=ACTION_GH_TOKEN export ACTION_GH_TOKEN=$(cat /run/sec && echo "#####################################" \ && cp /usr/local/lib/python3.9/dist-packages/jep/jep-4.2.0.jar /opt/IPED/iped/lib/jep-4.0.3.jar \ && echo "#####################################" \ + && echo "Configure TaskBridge-Scripts" \ + && echo "#####################################" \ + && sed -i -e 's/<\/task>/<\/task>\n <\/task>/' /opt/IPED/iped/conf/TaskInstaller.xml \ + && sed -i -e 's/<\/task>/<\/task>\n <\/task>\n <\/task>/' /opt/IPED/iped/conf/TaskInstaller.xml \ + && git clone https://github.com/hilderonny/iped-audiotranslatetask.git \ + && git clone https://github.com/hilderonny/iped-virusscantask.git \ + && git clone https://github.com/hilderonny/iped-imageclassificationtask.git \ + && cp iped-audiotranslatetask/conf/AudioTranslation.txt /opt/IPED/iped/conf/AudioTranslation.txt \ + && cp iped-audiotranslatetask/scripts/tasks/AudioTranslateTask.py /opt/IPED/iped/scripts/tasks/AudioTranslateTask.py \ + && cp iped-imageclassificationtask/conf/ImageClassification.txt /opt/IPED/iped/conf/ImageClassification.txt \ + && cp iped-imageclassificationtask/scripts/tasks/ImageClassificationTask.py /opt/IPED/iped/scripts/tasks/ImageClassificationTask.py \ + && cp iped-virusscantask/conf/VirusScanning.txt /opt/IPED/iped/conf/VirusScanning.txt \ + && cp iped-virusscantask/scripts/tasks/VirusScanningTask.py /opt/IPED/iped/scripts/tasks/VirusScanningTask.py \ + && echo "\nenableAudioTranslation = false" >> /opt/IPED/iped/IPEDConfig.txt \ + && echo "\nenableVirusScanning = false" >> /opt/IPED/iped/IPEDConfig.txt \ + && echo "\nenableImageClassification = false" >> /opt/IPED/iped/IPEDConfig.txt \ + && pip install --no-cache-dir requests \ + && echo "#####################################" \ && echo "Cleaning UP the container " \ && echo "#####################################" \ && rm -rfv ${PKGTMPDIR}/* && apt-get clean && rm -rfv /var/lib/apt/lists/* diff --git a/entrypoint.sh b/entrypoint.sh index 2aec391..a257091 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,54 +4,46 @@ PHOTODNA=false HASHESDB=false COUNTRY='BR' - - echo -n Populating IPED plugins directory with extra plugins... -if [ -d /mnt/plugins ] && [ ! -z "$(ls /mnt/plugins)" ] -then - ORIGDIR=${PWD} - cd /opt/IPED/plugins/ && find /mnt/plugins -type f \ - | xargs -I% sh -c 'ln -s "$@" > /dev/null 2>&1 && echo -n $@[OK]...|| echo -n $@[FAILED]...' _ % +if [ -d /mnt/plugins ] && [ ! -z "$(ls /mnt/plugins)" ]; then + ORIGDIR=${PWD} + cd /opt/IPED/plugins/ && find /mnt/plugins -type f | + xargs -I% sh -c 'ln -s "$@" > /dev/null 2>&1 && echo -n $@[OK]...|| echo -n $@[FAILED]...' _ % cd ${ORIGDIR} echo "Done." fi -if [ ! -z "$(ls /opt/IPED/plugins/ | grep -i photodna | grep -i '\.jar$' )" ] -then +if [ ! -z "$(ls /opt/IPED/plugins/ | grep -i photodna | grep -i '\.jar$')" ]; then PHOTODNA=true - echo -n Setting PhotoDNA related flags to $PHOTODNA... && \ - sed -i -e "s/enablePhotoDNA =.*/enablePhotoDNA = $PHOTODNA/" /opt/IPED/iped/IPEDConfig.txt && \ - sed -i -e "s/enablePhotoDNALookup =.*/enablePhotoDNALookup = $PHOTODNA/" /opt/IPED/iped/IPEDConfig.txt && \ - echo Done. || echo Failed. + echo -n Setting PhotoDNA related flags to $PHOTODNA... && + sed -i -e "s/enablePhotoDNA =.*/enablePhotoDNA = $PHOTODNA/" /opt/IPED/iped/IPEDConfig.txt && + sed -i -e "s/enablePhotoDNALookup =.*/enablePhotoDNALookup = $PHOTODNA/" /opt/IPED/iped/IPEDConfig.txt && + echo Done. || echo Failed. fi +if [ -f /mnt/hashesdb/iped-hashes.db ]; then + HASHESDB=true -if [ -f /mnt/hashesdb/iped-hashes.db ] -then - HASHESDB=true - - echo -n Setting HASHDB related flags to $HASHESDB... && \ - sed -i -e "s/enableHashDBLookup =.*/enableHashDBLookup = $HASHESDB/" /opt/IPED/iped/IPEDConfig.txt && \ - sed -i -e "s/enableLedCarving =.*/enableLedCarving = $HASHESDB/" /opt/IPED/iped/IPEDConfig.txt && \ - echo Done. || echo Failed. + echo -n Setting HASHDB related flags to $HASHESDB... && + sed -i -e "s/enableHashDBLookup =.*/enableHashDBLookup = $HASHESDB/" /opt/IPED/iped/IPEDConfig.txt && + sed -i -e "s/enableLedCarving =.*/enableLedCarving = $HASHESDB/" /opt/IPED/iped/IPEDConfig.txt && + echo Done. || echo Failed. # check if HASHESDBONTMP is setted, if it is, copy it to tmp dir # can be used in cases that hashesdb in on the network and the only way # to accelerate things is to put it on tmpdir, that is mandatory to be local - if [ "$HASHESDBONTMP" == "true" ] - then - echo -n "Copying iped-hashes.db to /mnt/ipedtmp..." && \ - cp -p --update /mnt/hashesdb/iped-hashes.db /mnt/ipedtmp/ && echo -n OK... && \ - echo -n "Updating config..." && \ - sed -i -e "s/hashesDB =.*/hashesDB = \/mnt\/ipedtmp\/iped-hashes.db/" /opt/IPED/iped/LocalConfig.txt && \ - echo OK. || -n echo Failed. + if [ "$HASHESDBONTMP" == "true" ]; then + echo -n "Copying iped-hashes.db to /mnt/ipedtmp..." && + cp -p --update /mnt/hashesdb/iped-hashes.db /mnt/ipedtmp/ && echo -n OK... && + echo -n "Updating config..." && + sed -i -e "s/hashesDB =.*/hashesDB = \/mnt\/ipedtmp\/iped-hashes.db/" /opt/IPED/iped/LocalConfig.txt && + echo OK. || -n echo Failed. fi fi - # Custom flags to be used to modify configuration on runtime # LocalConfig.txt variables (with iped_ prefix) for v in \ @@ -64,15 +56,22 @@ for v in \ iped_tskJarPath \ iped_mplayerPath \ iped_pluginFolder \ - iped_regripperFolder -do + iped_regripperFolder; do echo ${v}=${!v} - if [ "${!v}" ] - then + if [ "${!v}" ]; then sed -i -e "s|.*${v#iped_} =.*|${v#iped_} = ${!v}|" /opt/IPED/iped/LocalConfig.txt fi done +if [ $iped_taskBridgeUrl ]; then + v="iped_taskBridgeUrl" + echo "iped_taskBridgeUrl = $iped_taskBridgeUrl" + sed -i -e "s|.*${v#iped_} =.*|${v#iped_} = ${!v}|" /opt/IPED/iped/conf/AudioTranslation.txt + sed -i -e "s|.*${v#iped_} =.*|${v#iped_} = ${!v}|" /opt/IPED/iped/conf/ImageClassification.txt + sed -i -e "s|.*${v#iped_} =.*|${v#iped_} = ${!v}|" /opt/IPED/iped/conf/VirusScanning.txt + +fi + # IPEDConfig.txt variables (with iped_ prefix) for v in \ iped_enableHash \ @@ -108,65 +107,61 @@ for v in \ iped_enableVideoThumbs \ iped_enableDocThumbs \ iped_enableHTMLReport \ - iped_enableOCR -do + iped_enableOCR \ + iped_enableVirusScanning \ + iped_enableAudioTranslation \ + iped_enableImageClassification; do echo ${v}=${!v} - if [ "${!v}" ] - then + if [ "${!v}" ]; then sed -i -e "s|.*${v#iped_} =.*|${v#iped_} = ${!v}|" /opt/IPED/iped/IPEDConfig.txt fi done # IPED variables setting on the config dir (with iped_ prefix). # supportedMimes, host and port are repeatedly used on configs -# so they were removed. IPED Variables with dots cannot be used +# so they were removed. IPED Variables with dots cannot be used # as environment variables, removed also -for v in $( for file in $( find /opt/IPED/iped/conf/ -type f | grep Config.txt \ - | grep -v -i regex); do grep "=" $file | grep -v "^host =" \ - | grep -v "^port = " | cut -d "=" -f 1 \ - | grep -v "\." | grep -v "^#" | grep -v supportedMimes \ - | awk '{ if ($0 != "\r" ) {print "iped_"$0;} }';\ - done ) -do +for v in $(for file in $(find /opt/IPED/iped/conf/ -type f | grep Config.txt | + grep -v -i regex); do + grep "=" $file | grep -v "^host =" | + grep -v "^port = " | cut -d "=" -f 1 | + grep -v "\." | grep -v "^#" | grep -v supportedMimes | + awk '{ if ($0 != "\r" ) {print "iped_"$0;} }' +done); do echo ${v}=${!v} - if [ "${!v}" ] - then - find /opt/IPED/iped/conf/ -type f | grep Config.txt | grep -v -i regex | xargs sed -i -e "s|${v#iped_} =.*|${v#iped_} = ${!v}|" + if [ "${!v}" ]; then + find /opt/IPED/iped/conf/ -type f | grep Config.txt | grep -v -i regex | xargs sed -i -e "s|${v#iped_} =.*|${v#iped_} = ${!v}|" fi done echo Setting GraphConfig... for v in \ - iped_phone_region -do + iped_phone_region; do echo ${v}=${!v} - if [ "${!v}" ] - then - sed -i -e "s|.*\"$(echo ${v#iped_}| sed 's/_/-/g')\":.*|\"$(echo ${v#iped_}| sed 's/_/-/g')\":\"${!v}\",|" /opt/IPED/iped/conf/GraphConfig.json - else - sed -i -e "s|.*\"$(echo ${v#iped_}| sed 's/_/-/g')\":.*|\"$(echo ${v#iped_}| sed 's/_/-/g')\":\"${COUNTRY}\",|" /opt/IPED/iped/conf/GraphConfig.json + if [ "${!v}" ]; then + sed -i -e "s|.*\"$(echo ${v#iped_} | sed 's/_/-/g')\":.*|\"$(echo ${v#iped_} | sed 's/_/-/g')\":\"${!v}\",|" /opt/IPED/iped/conf/GraphConfig.json + else + sed -i -e "s|.*\"$(echo ${v#iped_} | sed 's/_/-/g')\":.*|\"$(echo ${v#iped_} | sed 's/_/-/g')\":\"${COUNTRY}\",|" /opt/IPED/iped/conf/GraphConfig.json fi done - # # Test for UID presence and, if exist, change the execution for this user id -# -if [ "${USERID}" ] -then - echo -n "Adding user for command execution..." && \ - useradd --uid ${USERID} -U tmpuser -m && echo "user added with UID ${USERID}." && \ - echo -n "Creating mplayer config on user environment..." && \ - sudo -u tmpuser mplayer >/dev/null 2>&1 && echo "OK" && \ - echo -n "Configuring cache of tmpuser ..." && \ - ln -fs /root/.cache /home/tmpuser/.cache && \ - chown -RL tmpuser:tmpuser /root/.cache && chmod +x /root && \ - echo "Executing command as UID $USERID..." && \ - sudo -u tmpuser --chdir=${PWD} --preserve-env=SAL_USE_VCLPLUGIN,JAVA_HOME,LD_LIBRARY_PATH,IPED_VERSION $@ || \ - echo "Running as UID $USERID Failed." - -else +# +if [ "${USERID}" ]; then + echo -n "Adding user for command execution..." && + useradd --uid ${USERID} -U tmpuser -m && echo "user added with UID ${USERID}." && + echo -n "Creating mplayer config on user environment..." && + sudo -u tmpuser mplayer >/dev/null 2>&1 && echo "OK" && + echo -n "Configuring cache of tmpuser ..." && + ln -fs /root/.cache /home/tmpuser/.cache && + chown -RL tmpuser:tmpuser /root/.cache && chmod +x /root && + echo "Executing command as UID $USERID..." && + sudo -u tmpuser --chdir=${PWD} --preserve-env=SAL_USE_VCLPLUGIN,JAVA_HOME,LD_LIBRARY_PATH,IPED_VERSION $@ || + echo "Running as UID $USERID Failed." + +else # no arguments = bash, otherwise exec then echo "Executing command as ROOT..." exec "$@" From c35e521fd4f242d14ad2ff09381150b20b510d4c Mon Sep 17 00:00:00 2001 From: mobab-th <4n6linux@gmail.com> Date: Tue, 11 Feb 2025 14:01:31 +0100 Subject: [PATCH 2/2] Update Dockerfile.processor --- Dockerfile.processor | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.processor b/Dockerfile.processor index adf8ee3..679be2d 100644 --- a/Dockerfile.processor +++ b/Dockerfile.processor @@ -1,4 +1,4 @@ -FROM localhost/iped:dependencies +FROM ipeddocker/iped:dependencies ARG SNAPSHOT=false IPED_RELEASE_VERSION=4.2.0 SNAPSHOT_WORKFLOW_ID=12279329706 PKGTMPDIR=/tmp/pkgs @@ -28,7 +28,6 @@ RUN --mount=type=secret,id=ACTION_GH_TOKEN export ACTION_GH_TOKEN=$(cat /run/sec curl -L https://github.com/sepinf-inc/IPED/releases/download/$IPED_RELEASE_VERSION/IPED-${IPED_RELEASE_VERSION}_plus_java_plugins.zip --output ${PKGTMPDIR}/iped.zip && \ unzip ${PKGTMPDIR}/iped.zip ;\ fi \ - && unzip ${PKGTMPDIR}/iped.zip \ && echo "Creating IPED simbolic link..." \ && ls | grep "iped-" | xargs -i sh -c 'ln -s "{}" iped' \ && echo "#####################################" \