From e3e20a7b07f34fcbfde3a26d09a75fc27e3a2489 Mon Sep 17 00:00:00 2001 From: adamoutler Date: Wed, 15 Feb 2023 03:26:12 +0000 Subject: [PATCH] Refactoring and linting --- .devcontainer/devcontainer.json | 5 +- .vscode/extensions.json | 3 +- .vscode/launch.json | 4 +- .vscode/settings.json | 7 ++- Dockerfile | 4 +- Jenkinsfile | 50 ++++++--------- src/aidgaf/aidgaf-server/__init__.py | 7 --- src/aidgaf/aidgaf-server/__main__.py | 5 -- src/aidgaf/aidgaf-server/const.py | 4 -- src/aidgaf/aidgaf_server/__init__.py | 7 +++ src/aidgaf/aidgaf_server/__main__.py | 5 ++ src/aidgaf/aidgaf_server/const.py | 10 +++ .../hash_calculator.py | 10 +-- .../{aidgaf-server => aidgaf_server}/idgaf.py | 62 +++++++++++-------- .../{aidgaf-server => aidgaf_server}/merge.py | 12 ++-- .../security.py | 10 +-- .../server.py | 44 +++++++------ .../settings.py | 5 +- 18 files changed, 134 insertions(+), 120 deletions(-) delete mode 100644 src/aidgaf/aidgaf-server/__init__.py delete mode 100644 src/aidgaf/aidgaf-server/__main__.py delete mode 100644 src/aidgaf/aidgaf-server/const.py create mode 100644 src/aidgaf/aidgaf_server/__init__.py create mode 100644 src/aidgaf/aidgaf_server/__main__.py create mode 100644 src/aidgaf/aidgaf_server/const.py rename src/aidgaf/{aidgaf-server => aidgaf_server}/hash_calculator.py (67%) rename src/aidgaf/{aidgaf-server => aidgaf_server}/idgaf.py (65%) rename src/aidgaf/{aidgaf-server => aidgaf_server}/merge.py (82%) rename src/aidgaf/{aidgaf-server => aidgaf_server}/security.py (92%) rename src/aidgaf/{aidgaf-server => aidgaf_server}/server.py (86%) rename src/aidgaf/{aidgaf-server => aidgaf_server}/settings.py (93%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 62f5cf5..5087621 100755 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,10 +13,9 @@ "extensions": [ "pomdtr.secrets", "ms-python.python", - "donjayamanne.python-extension-pack", - "ivory-lab.jenkinsfile-support", "njpwerner.autodocstring", - "KevinRose.vsc-python-indent" + "NicolasVuillamy.vscode-groovy-lint", + "ms-python.pylint" ] } } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index f7cafaf..dfe1fe6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,5 +1,6 @@ { "recommendations": [ - "pomdtr.secrets" + "pomdtr.secrets", + "nicolasvuillamy.vscode-groovy-lint" ] } \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index d4bed5b..36ba567 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Start Server", "type": "python", "request": "launch", - "program": "src/aidgaf/aidgaf-server/server.py", + "program": "src/aidgaf/aidgaf_server/server.py", "console": "integratedTerminal", "justMyCode": true }, @@ -16,7 +16,7 @@ "name": "IDGAF", "type": "python", "request": "launch", - "program": "src/aidgaf/aidgaf-server/idgaf.py", + "program": "src/aidgaf/aidgaf_server/idgaf.py", "console": "integratedTerminal", "justMyCode": true }, diff --git a/.vscode/settings.json b/.vscode/settings.json index 0ac257d..a7f8210 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,10 @@ { "secrets.enabledFolders": [ "aidgaf" - ] + ], + "files.associations": { + "[Jj]enkinsfile*": "groovy" + }, + "python.linting.pylintPath": "/usr/local/python/current/bin/python" + } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3148e48..121a85a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,6 @@ FROM alpine:latest RUN apk add python3 py3-pip \ && pip3 install openai\ && mkdir /app -COPY src/aidgaf /app +COPY src/aidgaf /app/aidgaf EXPOSE 8087 -ENTRYPOINT ["/usr/bin/python","/app/aidgaf-server/__main__.py",">/dev/stdout"] +ENTRYPOINT ["/usr/bin/python","/app/aidgaf/aidgaf_server/__main__.py",">/dev/stdout"] diff --git a/Jenkinsfile b/Jenkinsfile index 2ef1be0..515fd71 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,5 @@ -#! /bin/groovy +#!/bin/groovy +/* groovylint-disable CompileStatic, DuplicateStringLiteral, LineLength */ pipeline { agent { docker { @@ -9,18 +10,14 @@ pipeline { } } - - - - stages { - stage('Prepare Environment'){ - steps{ - sh "#!/bin/sh \n" + + stage('Prepare Environment') { + steps { + sh '#!/bin/sh \n' + 'id; apk add docker openrc git' } } - stage('Obtain Source'){ + stage('Obtain Source') { steps { git branch: 'main', url: 'https://git.adamoutler.com/aoutler/aidgaf-server.git' } @@ -28,42 +25,35 @@ pipeline { stage('Build in docker') { steps { // Get some code from a Git repository - sh "#!/bin/sh \n" + + sh '#!/bin/sh \n' + 'docker build -t aidgaf .' } } - stage ("setup credentials"){ - steps{ + stage('setup credentials') { + steps { withCredentials([ sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey', usernameVariable: 'user')]) { - sh "#!/bin/sh \n" + + sh '#!/bin/sh \n' + 'set +e; docker stop aidgaf-server||echo machine stopped; docker rm aidgaf-server||echo machine does not exist; set -e' } - } } - stage ('export docker container'){ + stage('export docker container') { steps { - - sh "#!/bin/sh \n" + + sh '#!/bin/sh \n' + 'set +e; docker stop aidgaf-server||echo machine stopped; docker rm aidgaf-server||echo machine does not exist; set -e' withCredentials([ string(credentialsId: 'OpenAI-API-Token', variable: 'OPEN_AI_TOKEN'), string(credentialsId: 'PapaHashingSecret', variable: 'PAPA_HASH'), - sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey', usernameVariable: 'user') + string(credentialsId: 'PapaAsyncUrl', variable: 'ASYNC_URL'), + sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey') ]) { - sh "#!/bin/sh \n" + - 'mkdir -p ~/.ssh; cp "${sshkey}" ~/.ssh/id_rsa' - sh "#!/bin/sh \n" + - 'docker run --name=aidgaf-server -eSERVERPORT=8087 -eHOSTNAME=0.0.0.0 -eHASHKEY="${PAPA_HASH}" -eAPIKEY="${OPEN_AI_TOKEN}" -p8087:8087 -d --restart=always aidgaf' - } + sh '#!/bin/sh \n' + + 'mkdir -p ~/.ssh; cp "$sshkey" ~/.ssh/id_rsa' + sh '#!/bin/sh \n' + + /* groovylint-disable-next-line GStringExpressionWithinString */ + 'docker run --name=aidgaf-server -eSERVERPORT=8087 -eHOSTNAME=0.0.0.0 -eHASHKEY="${PAPA_HASH}" -eAPIKEY="${OPEN_AI_TOKEN}" -eASYNC_METHOD="PATCH" -eASYNC_URL="${} -p8087:8087 -d --restart=always aidgaf' + } } } - - - // To run Maven on a Windows agent, use - // bat "mvn -Dmaven.test.failure.ignore=true clean package" - - - } } diff --git a/src/aidgaf/aidgaf-server/__init__.py b/src/aidgaf/aidgaf-server/__init__.py deleted file mode 100644 index 1279e6a..0000000 --- a/src/aidgaf/aidgaf-server/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" aidgaf-server. """ - -""" The name of the package""" -name = 'aidgaf' - -""" The version of the package""" -version='0.1.0' \ No newline at end of file diff --git a/src/aidgaf/aidgaf-server/__main__.py b/src/aidgaf/aidgaf-server/__main__.py deleted file mode 100644 index 38bba93..0000000 --- a/src/aidgaf/aidgaf-server/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" aidgaf-server entry point. """ -import server - -""" The main entry point for the aidgaf-server package.""" -server.main() \ No newline at end of file diff --git a/src/aidgaf/aidgaf-server/const.py b/src/aidgaf/aidgaf-server/const.py deleted file mode 100644 index 5bc624c..0000000 --- a/src/aidgaf/aidgaf-server/const.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Constants for the aidgaf-server package.""" - -""" The encoding to use for strings. """ -UTF8="utf-8" \ No newline at end of file diff --git a/src/aidgaf/aidgaf_server/__init__.py b/src/aidgaf/aidgaf_server/__init__.py new file mode 100644 index 0000000..989e692 --- /dev/null +++ b/src/aidgaf/aidgaf_server/__init__.py @@ -0,0 +1,7 @@ +""" aidgaf-server. """ + +# The name of the package +name = 'aidgaf_server' # pylint: disable=C0103 + +# The version of the package +version = '0.1.0' # pylint: disable=C0103 diff --git a/src/aidgaf/aidgaf_server/__main__.py b/src/aidgaf/aidgaf_server/__main__.py new file mode 100644 index 0000000..b244abe --- /dev/null +++ b/src/aidgaf/aidgaf_server/__main__.py @@ -0,0 +1,5 @@ +""" aidgaf-server entry point. """ +import server + +# The main entry point for the aidgaf-server package. +server.main() diff --git a/src/aidgaf/aidgaf_server/const.py b/src/aidgaf/aidgaf_server/const.py new file mode 100644 index 0000000..e366f13 --- /dev/null +++ b/src/aidgaf/aidgaf_server/const.py @@ -0,0 +1,10 @@ +"""Constants for the aidgaf-server package.""" + +UTF8="utf-8" +""" The encoding to use for strings. """ + +OPENAI_TIMEOUT=30 +""" The timeout for OpenAI requests. """ + +REPORTING_TIMEOUT=10 +""" The timeout for async reporting requests. """ \ No newline at end of file diff --git a/src/aidgaf/aidgaf-server/hash_calculator.py b/src/aidgaf/aidgaf_server/hash_calculator.py similarity index 67% rename from src/aidgaf/aidgaf-server/hash_calculator.py rename to src/aidgaf/aidgaf_server/hash_calculator.py index a0d726e..9ff6017 100644 --- a/src/aidgaf/aidgaf-server/hash_calculator.py +++ b/src/aidgaf/aidgaf_server/hash_calculator.py @@ -1,6 +1,8 @@ +""" This module contains the function to calculate the hash of a value and a secret.""" import hashlib -def calculate_hash(value:bytes, secret:bytes)->str: + +def calculate_hash(value: bytes, secret: bytes) -> str: """ This function calculates the hash of a value and a secret. It is used to verify that the message is from the server. The hash is calculated using the SHA512 algorithm. The hash is returned as a @@ -13,6 +15,6 @@ def calculate_hash(value:bytes, secret:bytes)->str: returns: The hash of the value and secret. This is a hex string. """ - m = hashlib.sha512() - m.update(b"".join([value,secret])) - return m.hexdigest() + sha512 = hashlib.sha512() + sha512.update(b"".join([value, secret])) + return sha512.hexdigest() diff --git a/src/aidgaf/aidgaf-server/idgaf.py b/src/aidgaf/aidgaf_server/idgaf.py similarity index 65% rename from src/aidgaf/aidgaf-server/idgaf.py rename to src/aidgaf/aidgaf_server/idgaf.py index 3bb9c3b..2c286f3 100644 --- a/src/aidgaf/aidgaf-server/idgaf.py +++ b/src/aidgaf/aidgaf_server/idgaf.py @@ -1,61 +1,61 @@ """ This file contains the logic for the IDGAF server. It can be used to make a stand-alone IDGAF server, or it can be used as a module in a larger application. """ +from datetime import datetime import json import random -import security -import re import requests -from time import time -from datetime import datetime import settings +from const import OPENAI_TIMEOUT -""" The URL for the OpenAI API. """ URL = "https://api.openai.com/v1/completions" +""" The URL for the OpenAI API. """ -""" The data to send to OpenAI. """ DATA = {"model": settings.OPEN_AI_COMPLETION_MODEL, "prompt": settings.PROMPTS[0], "temperature": settings.TEMPERATURE, "max_tokens": settings.OPEN_AI_MAX_TOKENS } +""" The data to send to OpenAI. """ -""" The headers to send to OpenAI. """ request_headers = {"Authorization": "Bearer " + settings.APIKEY, "Content-Type": "application/json"} +""" The headers to send to OpenAI. """ -def get_response_base_object(text:str) -> dict: + +def get_response_base_object(text: str) -> dict: """ This is used to create the response object for the server. Parameters: text: The text to return to the client. Returns: A dictionary containing the response object. - """ - resultObject = {} - resultObject["message"] = {} - resultObject["service"] = "AIDGAF Server" - resultObject["message"]["data"] = {} - resultObject["message"]["data"]["resultObject"] = text - resultObject["timestamp"] = datetime.utcnow().timestamp() - return resultObject + """ + result_object = {} + result_object["message"] = {} + result_object["service"] = "AIDGAF Server" + result_object["message"]["data"] = {} + result_object["message"]["data"]["resultObject"] = text + result_object["timestamp"] = datetime.utcnow().timestamp() + return result_object -def parse_idgaf_request(command)->[int, dict]: +def parse_idgaf_request(idgaf_command) -> [int, dict]: """ This function handles the IDGAF command. It will return a response object. Parameters: command: The command object received from the client. Returns: A tuple containing the status code and the response object. """ - the_data = get_prompt(command) + the_data = get_prompt(idgaf_command) response = get_gpt_response(the_data) try: response_text = response.json()['choices'][0]['text'].strip() - except (KeyError): - response_text = gpt_response.text + except KeyError: + response_text = response.text obj = get_response_base_object(response_text) return [response.status_code, obj] + def get_gpt_response(data) -> requests.Response: """ This function communicates with OpenAI and returns a response object. Parameters: @@ -63,11 +63,14 @@ def get_gpt_response(data) -> requests.Response: Returns: The response object from OpenAI. """ - gpt_response = requests.post(URL, json=data, headers=request_headers) + gpt_response = requests.post( + url=URL, json=data, headers=request_headers, timeout=OPENAI_TIMEOUT) return gpt_response -def get_prompt(command)->dict: - """ Selects a prompt from the PROMPTS list and replaces the USERNAME placeholder with the username. + +def get_prompt(command) -> dict: + """ Selects a prompt from the PROMPTS list and replaces the USERNAME placeholder + with the username. Parameters: command: The command object received from the client. Returns: @@ -83,9 +86,14 @@ def get_prompt(command)->dict: return the_data +# This function is for testing the IDGAF capabilities without a server. if __name__ == "__main__": - """ This function is for testing the IDGAF capabilities without a server. """ - value = '{"service":"papa","message":{"command":"aidgaf","data":{"username":"AdamOutler"},"timestamp":1675725191},"hash":"1bc73914478835d03f9ebdfb46328321d2bb656647e2876d6f162cc1860607fcfca8d825c48e390a6a254ee0835c8a4fe5f9a25795a3a0880ae5a23e9c132cf2"}' - command = json.loads(value) - [code, result] = parse_idgaf_request(command) + INPUT = '''{"service":"papa","message": + {"command":"aidgaf","data":{"username":"AdamOutler"}, + "timestamp":1675725191}, + "hash":"1bc73914478835d03f9ebdfb46328321d2bb656647e28 + 76d6f162cc1860607fcfca8d825c48e390a6a254ee0835c8a4fe5f + 9a25795a3a0880ae5a23e9c132cf2"}''' + test_command = json.loads(INPUT) + [code, result] = parse_idgaf_request(test_command) print(result) diff --git a/src/aidgaf/aidgaf-server/merge.py b/src/aidgaf/aidgaf_server/merge.py similarity index 82% rename from src/aidgaf/aidgaf-server/merge.py rename to src/aidgaf/aidgaf_server/merge.py index 92fbbb2..3113c24 100644 --- a/src/aidgaf/aidgaf-server/merge.py +++ b/src/aidgaf/aidgaf_server/merge.py @@ -1,18 +1,18 @@ """This module contains the merge_dict function.""" -from functools import reduce def merge_dict_no_overwrite(base_dictionary, other_dictionary, location=None)->dict: """ This function merges two dictionaries. If the same key exists in both dictionaries, the value from the other_dictionary will be used. This function will recurse into nested dictionaries. If the same key exists in both dictionaries, and the value is a dictionary, - the function will recurse into the nested dictionary. If the same key exists in both dictionaries, - and the value is not a dictionary, the value from the base_dictionary will be used. + the function will recurse into the nested dictionary. If the same key exists in both + dictionaries, and the value is not a dictionary, the value from the base_dictionary will + be used. Parameters: base_dictionary: The dictionary to merge the other_dictionary into. - other_dictionary: The dictionary to merge keys from. if the same key exists in both dictionaries, - the value from the base_dictionary will be used. + other_dictionary: The dictionary to merge keys from. if the same key exists in both + dictionaries, the value from the base_dictionary will be used. location: leave blank. This is used to track the location of the key in the dictionary. Returns: The merged dictionary. @@ -29,5 +29,3 @@ def merge_dict_no_overwrite(base_dictionary, other_dictionary, location=None)->d else: base_dictionary[key] = other_dictionary[key] return base_dictionary - - diff --git a/src/aidgaf/aidgaf-server/security.py b/src/aidgaf/aidgaf_server/security.py similarity index 92% rename from src/aidgaf/aidgaf-server/security.py rename to src/aidgaf/aidgaf_server/security.py index edd3394..0ac39cd 100644 --- a/src/aidgaf/aidgaf-server/security.py +++ b/src/aidgaf/aidgaf_server/security.py @@ -13,9 +13,9 @@ def perform_hash_checks(str_request)->bool: str_request: The request body, as a string. Returns: True if the message is valid, False otherwise.""" - hash = get_message_hash(str_request) + my_hash = get_message_hash(str_request) - if hash not in str_request: + if my_hash not in str_request: print("Error: hash not match") return False if not verify_message_time(str_request): @@ -30,14 +30,14 @@ def get_message_hash(json_command) -> str: json_command: The JSON command to hash. Returns: The hash of the message, as a string. """ - + strip1 = re.sub(".*\"message\":", "", json_command, 1) - if ("\"hash\":" in strip1): + if "\"hash\":" in strip1: strip2 = re.sub(',\"hash\":.*', '', strip1) else: strip2 = strip1[:-1] json_value = bytes(strip2, UTF8) - if (settings.HASHKEY is not None): + if settings.HASHKEY is not None: hash_value = hash_calculator.calculate_hash( json_value, settings.HASHKEY) return hash_value diff --git a/src/aidgaf/aidgaf-server/server.py b/src/aidgaf/aidgaf_server/server.py similarity index 86% rename from src/aidgaf/aidgaf-server/server.py rename to src/aidgaf/aidgaf_server/server.py index afdac23..524112d 100644 --- a/src/aidgaf/aidgaf-server/server.py +++ b/src/aidgaf/aidgaf_server/server.py @@ -1,16 +1,15 @@ """ aidgaf-server - A simple server to handle requests from the aidgaf client. This server will communicate with OpenAI to generate responses to the client.""" +from http.server import BaseHTTPRequestHandler, HTTPServer import json -import random import threading import requests -import merge -from http.server import BaseHTTPRequestHandler, HTTPServer +import merge import idgaf import settings import security -from const import UTF8 +from const import UTF8, REPORTING_TIMEOUT default_request_body = b'{"message":{"command":"aidgaf","data":{"username":"AdamOutler"}}}' request_headers = {} @@ -23,13 +22,18 @@ class IDGAFServer(BaseHTTPRequestHandler): self.send_response(418) self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write(bytes("", UTF8)) + self.wfile.write(bytes("HTTP 408\nI'm a teapot", UTF8)) + self.close_connection def do_PATCH(self): """ This function handles PATCH requests. """ body = self.get_body().decode(UTF8) - if not perform_sanity_checks(body): + check_result=perform_sanity_checks(body) + if check_result: self.send_response(403) + self.end_headers() + self.wfile.write(bytes(check_result, UTF8)) + self.close_connection return command = json.loads(body) self.do_request_handling(command) @@ -69,21 +73,20 @@ class IDGAFServer(BaseHTTPRequestHandler): merged['service']='JavaShark' data=json.dumps(merged) print(str(code)+": "+data) - + if (code != 200): return if (settings.ASYNC_METHOD == "POST"): - requests.post(settings.ASYNC_URL, data=data, headers=request_headers) + requests.post(settings.ASYNC_URL, data=data, headers=request_headers, + timeout=REPORTING_TIMEOUT) elif (settings.ASYNC_METHOD == "PUT"): - requests.put(settings.ASYNC_URL, data=data, headers=request_headers) + requests.put(settings.ASYNC_URL, data=data, headers=request_headers, + timeout=REPORTING_TIMEOUT) elif (settings.ASYNC_METHOD == "PATCH"): - resp=requests.patch(settings.ASYNC_URL, data=data, headers=request_headers) + resp=requests.patch(settings.ASYNC_URL, data=data, headers=request_headers, + timeout=REPORTING_TIMEOUT) print(str(resp)+resp.text) - - - - # gpt_response = requests.post(URL, json=data, headers=request_headers) @@ -124,9 +127,10 @@ class IDGAFServer(BaseHTTPRequestHandler): self.end_headers() print("sending:"+body) self.wfile.write(bytes(body, UTF8)) + self.close_connection -def perform_sanity_checks(str_request) -> bool: +def perform_sanity_checks(str_request) -> str: """ Performs a hash check on the message, and verifies the timestamp is valid. If either check fails, return False. Otherwise, return True. Parameters: @@ -135,13 +139,13 @@ def perform_sanity_checks(str_request) -> bool: True if the message is valid, False otherwise. """ if settings.HASHKEY is not None: hash = security.get_message_hash(str_request) - if hash not in str_request: + if settings.HASHKEY and hash not in str_request: print("Error: hash not match") - return False - if not security.verify_message_time(str_request): + return "Error: hash not match" + if settings.HASHKEY and not security.verify_message_time(str_request): print("Error: timestamp expired") - return False - return True + return "Error: timestamp expired" + return "" def main(): diff --git a/src/aidgaf/aidgaf-server/settings.py b/src/aidgaf/aidgaf_server/settings.py similarity index 93% rename from src/aidgaf/aidgaf-server/settings.py rename to src/aidgaf/aidgaf_server/settings.py index 2897081..b6d30c9 100644 --- a/src/aidgaf/aidgaf-server/settings.py +++ b/src/aidgaf/aidgaf_server/settings.py @@ -42,7 +42,8 @@ OPEN_AI_COMPLETION_MODEL = "text-davinci-003" TEMPERATURE = 0.7 """ The hash key for the server. Leave this blank if you don't want to use it. """ -HASHKEY = bytes(os.getenv('HASHKEY'), - UTF8) # shared secret for hmac of message +HASHKEY = bytes(os.getenv('HASHKEY') or "",UTF8) # shared secret for hmac of message +if (HASHKEY == ""): + HASHKEY=None """ The maximum age of a message in seconds. Only used if HASHKEY is set.""" MAX_MESSAGE_AGE = 600