BulkImageImportTest/checkImageFolder.sh at master · dcsr-datumo/BulkImageImportTest · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/bin/bash

: ${KNORA_BULK_ENDPOINT:=http://localhost:3333/v1/resources/xmlimport/}
: ${KNORA_USER:=root@example.com:test}

if [ "$#" -lt 2 ]; then
    echo -e "Usage: $0 images_folder image_xml_path [no_full_bulk]"
    echo "KNORA_BULK_ENDPOINT=$KNORA_BULK_ENDPOINT"
    echo "KNORA_USER=$KNORA_USER"
    echo "to set new BULK_ENDPOINT:"
    echo 'export KNORA_BULK_ENDPOINT="https://knora-nv.unil.ch/v1/resources/xmlimport/"'
    exit 1
fi

full_bulk=on
if [ "$#" -eq 3 ] && [ $3 == "no_full_bulk" ] ; then
    full_bulk=off
fi

echo "KNORA_BULK_ENDPOINT=$KNORA_BULK_ENDPOINT"
echo "KNORA_USER=$KNORA_USER"

BULK_HEADER="<?xml version=\"1.0\" encoding=\"UTF-8\"?><knoraXmlImport:resources xmlns=\"http://api.knora.org/ontology/0001/anything/xml-import/v1#\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://api.knora.org/ontology/0001/anything/xml-import/v1# p0001-anything.xsd\" xmlns:p0001-anything=\"http://api.knora.org/ontology/0001/anything/xml-import/v1#\" xmlns:knoraXmlImport=\"http://api.knora.org/ontology/knoraXmlImport/v1#\">"
BULK_FOOTER="</knoraXmlImport:resources>"
BULK_TEMPLATE_FILE="ThingPicture-template.xml"

warn_count=0
error_count=0
count_entries=0
files=($(find $1 -maxdepth 1 -type f -not -name ".*" | sort))
count_files=${#files[@]}
allbulk_file=/tmp/allbulk-$(date +%s)
image_xml_path=$2
echo ${BULK_HEADER} > $allbulk_file
echo "Found $count_files file(s)"

echo "--> Bulk Test BEGIN $(date)"
echo ""
fileCount=0
for (( i=0; i<${count_files}; i++ ));
do
    ((fileCount++))
    d=$(date +%s)
    file=${files[$i]}
    filename=$(basename -- "$file")
    extension="${filename##*.}"
    type=""
    type=$(grep $extension":" mimetypes | cut -d":" -f2)
    if [ "$type" == "" ]; then
        echo "    -${fileCount}/${count_files}-  file=$file"
        echo "    ### WARNING $file ignored, cannot find file type from mimetypes file"
        echo ""
        ((warn_count++))
    else
        bulk_file=/tmp/bulk-$d
        base_filename=$(basename $file)
        bulk=$(sed -e 's,@@FILE_PATH@@,'"${image_xml_path}/${base_filename}"',g' -e 's,@@MIMETYPE@@,'"$type"',g' -e 's,@@DATE_EPOCH@@,'"$d"',g'  ${BULK_TEMPLATE_FILE})
        echo "    -${fileCount}/${count_files}- bulk id=$d file=$file ($(date))"
        echo ${BULK_HEADER} > $bulk_file
        echo $bulk >> $bulk_file
        echo $bulk >> $allbulk_file
        echo ${BULK_FOOTER} >> $bulk_file
        cmd="curl -k -s -u ${KNORA_USER} -X POST -d @$bulk_file ${KNORA_BULK_ENDPOINT}http%3A%2F%2Frdfh.ch%2Fprojects%2F0001"
        echo "      CMD: $cmd"
        response=$($cmd)
        if [[ $response =~ .*createdResources.* ]] ; then
            ((count_entries++))
            echo "    ### SUCCESS $file"
        else
            echo ""
            echo "    "$(cat $bulk_file)
            echo "    ### ERROR $file"
            echo "    $response"
            ((error_count++))
        fi
        echo ""
        # wait 1s to avoid id collision based on seconds from epoch
        sleep 1
    fi
done
echo ""
echo "<-- Bulk Test END $(date)"

echo "#### FOUND ${count_entries} ENTRIES"

if [ $warn_count != 0 ]; then
    echo "#### $warn_count WARNING(S) FOUND: CHECK LOGS ABOVE"
else
    echo "#### NO WARNING FOUND"
fi
if [ $error_count != 0 ]; then
    echo "#### $error_count ERROR(S) FOUND: CHECK LOGS ABOVE"
else
    echo "#### NO ERROR FOUND"
fi

if [ $full_bulk == "on" ] ; then
    echo ""
    echo "--> full bulk with ${count_entries} entries - file=$allbulk_file ($(date))"
    echo ${BULK_FOOTER} >> $allbulk_file
    cmd="curl -k -s -u ${KNORA_USER} -X POST -d @$allbulk_file ${KNORA_BULK_ENDPOINT}http%3A%2F%2Frdfh.ch%2Fprojects%2F0001"
    echo " CMD: $cmd"
    response=$($cmd)
    if [[ $response =~ .*createdResources.* ]] ; then
    echo "    ### SUCCESS full bulk import: $allbulk_file"
    else
    echo ""
    echo "    ### ERROR full bulk import: $allbulk_file"
    echo "    $response"
    fi
    echo ""
    echo "<-- all bulk file=$allbulk_file ($(date))"
else
    echo "Full bulk disable"
fi