application.py

from __future__ import print_function
from flask import Flask, render_template, request, redirect, url_for

import json
import pygraphviz as pgv
import re
from lxml import etree
#replace StringIO
import io
#from io import BytesIO
#from io import StringIO
import urllib
#import urllib2
import urllib.request
import urllib3
import string
import sys
import os
import os.path
import natsort
import logging
from collections import OrderedDict
from werkzeug.middleware.proxy_fix import ProxyFix

from logging.config import dictConfig

# From https://flask.palletsprojects.com/en/2.3.x/logging/
dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
    }},
    'handlers': {'wsgi': {
        'class': 'logging.StreamHandler',
        'stream': 'ext://flask.logging.wsgi_errors_stream',
        'formatter': 'default'
    }},
    'root': {
        'level': 'WARN', # Adapt this to output other warnings
        'handlers': ['wsgi']
    }
})


app = Flask(__name__)
# Correctly use X-Forwarded-Proto in returned URLs to work in https reverse proxy
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)

os.environ['PATH'] = os.environ['PATH'] + ':/usr/local/bin'
os.environ['GV_FILE_PATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/images/')) + '/'

print('PATH: ' + os.environ['PATH'], file=sys.stderr)
print('GV_FILE_PATH: ' + os.environ['GV_FILE_PATH'], file=sys.stderr)

class TitleNotFoundException(Exception):
    pass

def getvar(varname):
    if varname in os.environ:
        return os.environ[varname]
    else:
        raise RuntimeError(f"{varname} not defined")        

def get_people():
    url = getvar("CAPX_URL") + "/api/skills/getAllGrouped"
    headers = {'x-api-key': getvar("CAPX_API_KEY")}
    request = urllib.request.Request(url, headers=headers)
    response = urllib.request.urlopen(request)
    data = json.loads(response.read())
    transformed_data = {}
    for persondata in data:
        transformed_data[persondata["name"]] = {
            "interests": [skill["name"] for skill in persondata["skills"]]
        }
    return transformed_data

# thanks to Colin Morris for adding this code originally
def get_skills_list():
    skills_list = {}
    json_results = get_people()
    for supervisor, data in json_results.items():
        for item in data["interests"]:
            if item not in skills_list:
                skills_list[item] = 1
            else:
                skills_list[item] += 1

    skills_list_new = OrderedDict(natsort.natsorted(skills_list.items()))
    return skills_list_new


def canonical_title(topic):
    """Query wikipedia to find the canonical title of a topic"""

    url = 'https://en.wikipedia.org/w/api.php'
    values = {
        'action': 'query',
        'titles': topic,
        'format': 'json',
    }

    data = urllib.parse.urlencode(values)
    data = data.encode('utf-8')

    request = urllib.request.Request(url, data)
    response = urllib.request.urlopen(request)
    json_response = response.read()
    json_result = json.loads(json_response)
    page = list(json_result['query']['pages'].values())[0]

    if "pageid" in page:
        canonical_title = page["title"]
    elif "missing" in page:
        raise TitleNotFoundException(
            f"Topic {topic} is not the title of a Wikipedia article")
    else:
        raise Exception("Unrecognised response from wikipedia in \
        canonical_title")

    return canonical_title


def get_graph_string(graph):
    #output = StringIO.StringIO()
    output = io.BytesIO()
    graph.draw(output, format = 'svg')
    svg = output.getvalue()
    output.close()

    svg_parser = etree.XMLParser()
    svg_obj = etree.fromstring(svg, svg_parser)

    svg_obj.attrib['width'] = '100%'
    del svg_obj.attrib['height']

    images = svg_obj.findall('.//{http://www.w3.org/2000/svg}image')

    # Convert all image href links to match server
    # For example, anonymous.png becomes static/images/anonymous.png
    # Have to do this because graphviz doesn't allow you to specify the
    # URL to an image, only the file path.
    for image in images:
        image_filename = image.attrib['{http://www.w3.org/1999/xlink}href']
        image_url = url_for('static', filename = 'images/' + image_filename)
        image.attrib['{http://www.w3.org/1999/xlink}href'] = image_url

    return etree.tostring(svg_obj, pretty_print = True).decode('utf-8')

def get_image_files():
    image_files = []
    image_dir = os.environ['GV_FILE_PATH']

    for root, sub_folders, files in os.walk(image_dir):
        for filename in files:
            actual_file_name = os.path.join(root, filename)
            if filename.endswith('.png'):
                image_files.append(filename)

    return image_files

def build_graph(name, results, topics):
    graph = pgv.AGraph(overlap = 'false', name = name)
    people = get_people()

    for person in results:
        person_names = person.lower().split()
        forename = person_names[0]
        surname = " ".join(person_names[1:])

        image_file = 'anonymous.png'
        image_files = get_image_files()

        for filename in image_files:
			#newstr = starturlsource[index+len(pattern):index+len(pattern)+17]
            if str.find(filename, surname) != -1 and str.find(filename, forename) != -1:
                image_file = '%s' % (filename)

        # check added for _ in name e.g. Anja Le_Blanc; that is: convert _ to space
        myperson= person.replace(' ', '\n')
        graph.add_node(person, label = myperson.replace('_' , ' '), fontname = 'Helvetica', fixedsize = True, imagescale = True, width = '1.5', height = '1.5', fontcolor = 'white', shape = 'circle', style = 'filled', color = '#303030', URL = url_for('show_person', name = person), image = image_file)

        interests = people[person]['interests']
        for interest in interests:
            if interest in topics:
                color = '#A02020FF'
                shape = 'ellipse'
            else:
                color = '#105060EE'
                shape = 'ellipse'

            label = re.sub('\(.*\)', '', interest)

            graph.add_node(interest, label = label, style = 'filled', fontname = 'Helvetica', shape = shape, color = color, fontcolor = 'white', URL = url_for('show_topic', name = interest))
            graph.add_edge(person, interest, color = '#00000050')

        if 'technologies' in people[person]:
            for technology in people[person]['technologies']:
                if technology in topics:
                    color = '#B01050FF'
                    shape = 'ellipse'
                else:
                    color = '#701050EE'
                    shape = 'ellipse'

                label = re.sub('\(.*\)', '', technology)

                graph.add_node(technology, label = label, style = 'filled', fontname = 'Helvetica', shape = shape, color = color, fontcolor = 'white')
                graph.add_edge(person, technology, color = '#00000050')

    graph.layout(prog = 'neato')

    return graph

def get_skills():
    """Returns the list of skills"""
    skills_list = get_skills_list()
    skills = [p for p in skills_list]
    return skills

@app.route('/')
def index():
    graph_name = 'UoM Research Software Engineers and their skills'

    results = set()
    topics = []

    people = get_people()

    for person in people:
        results.add(person)

    graph = build_graph(graph_name, results, topics)
    graph_str = get_graph_string(graph)
    interests_links = get_interests_links()

    return render_template('graph.html', name=graph_name, node_count=len(graph.nodes()), graph=graph_str, interests=interests_links, skills=get_skills())


def get_interests_links():
    interests_skills = get_skills_list()
    interests_links = ""
    index_link = url_for('index')
    for skill, count in interests_skills.items():
        interests_links += '<a href="' + index_link + 'topic/' + str(skill) + '" title="' + str(count) + ' records">' + str(skill) + '</a><br>'
    return interests_links


@app.route('/person/<name>')
@app.route('/person/')
@app.route('/person')
def show_person(name=None):
    if not name:
        interests_links = get_interests_links()
        return render_template('notfound.html', interests=interests_links, skills=get_skills())

    graph_name = name.replace('_', ' ') + "'s skills"

    results = set()
    topics = []

    results.add(name)

    graph = build_graph(graph_name, results, topics)
    if graph is False:
        pname=name
        interests_links = get_interests_links()
        return render_template('notfound.html', search_term=pname, interests=interests_links, skills=get_skills())
    graph_str = get_graph_string(graph)

    return render_template('graph.html', name=graph_name, node_count=len(graph.nodes()), graph=graph_str, skills=get_skills())


@app.route('/topic/<name>')
def show_topic(name):
    main_topic = name
    graph_name = 'UoM RSEs and their skills related to ' + name

    results = set()
    # We could implement a fuzzy search algorithm here, but it would make it hard to search
    # for things like "C" without adding options for "exact match" etc.
    topics = [skill for skill in get_skills() if name.lower() == skill.lower()]

    people = get_people()
    interests_links = get_interests_links()

    for person in people:
        for topic in topics:
            if topic in people[person]['interests']:
                results.add(person)

        if 'technologies' in people[person]:
            for topic in topics:
                if topic in people[person]['technologies']:
                    results.add(person)

    graph = build_graph(graph_name, results, topics)
    graph_str = get_graph_string(graph)
    return render_template('graph.html', name=graph_name, node_count=len(graph.nodes()), graph=graph_str, interests=interests_links, skills=get_skills())

# 2019-04-08 | New : Try to handle empty search
@app.route('/topic/')
def show_topic_notfound():
    interests_links = get_interests_links()
    return render_template('notfound.html', interests=interests_links, skills=get_skills())

@app.route('/topic-search')
def topic_search():
    topic = request.args.get('topic')
    return redirect(url_for('show_topic', name = topic))


if __name__ == "__main__":
    app.run()