aoutler/aidgaf-server
aoutler/aidgaf-server
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring and linting

Browse Source
This commit is contained in:
adamoutler 2023-02-15 03:26:12 +00:00
parent 2752702d17
commit e3e20a7b07
18 changed files with 134 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.devcontainer/devcontainer.json
View File

@ -13,10 +13,9 @@
"extensions": [
"pomdtr.secrets",
"ms-python.python",
"donjayamanne.python-extension-pack",
"ivory-lab.jenkinsfile-support",
"njpwerner.autodocstring",
"KevinRose.vsc-python-indent"
"NicolasVuillamy.vscode-groovy-lint",
"ms-python.pylint"
]
}
}

3
.vscode/extensions.json vendored
View File

@ -1,5 +1,6 @@
{
"recommendations": [
"pomdtr.secrets"
"pomdtr.secrets",
"nicolasvuillamy.vscode-groovy-lint"
]
}

4
.vscode/launch.json vendored
View File

@ -8,7 +8,7 @@
"name": "Start Server",
"type": "python",
"request": "launch",
"program": "src/aidgaf/aidgaf-server/server.py",
"program": "src/aidgaf/aidgaf_server/server.py",
"console": "integratedTerminal",
"justMyCode": true
},
@ -16,7 +16,7 @@
"name": "IDGAF",
"type": "python",
"request": "launch",
"program": "src/aidgaf/aidgaf-server/idgaf.py",
"program": "src/aidgaf/aidgaf_server/idgaf.py",
"console": "integratedTerminal",
"justMyCode": true
},

7
.vscode/settings.json vendored
View File

@ -1,5 +1,10 @@
{
"secrets.enabledFolders": [
"aidgaf"
]
],
"files.associations": {
"[Jj]enkinsfile*": "groovy"
},
"python.linting.pylintPath": "/usr/local/python/current/bin/python"
}

4
Dockerfile
View File

@ -17,6 +17,6 @@ FROM alpine:latest
RUN apk add python3 py3-pip \
&& pip3 install openai\
&& mkdir /app
COPY src/aidgaf /app
COPY src/aidgaf /app/aidgaf
EXPOSE 8087
ENTRYPOINT ["/usr/bin/python","/app/aidgaf-server/__main__.py",">/dev/stdout"]
ENTRYPOINT ["/usr/bin/python","/app/aidgaf/aidgaf_server/__main__.py",">/dev/stdout"]

48
Jenkinsfile vendored
View File

@ -1,4 +1,5 @@
#! /bin/groovy
#!/bin/groovy
/* groovylint-disable CompileStatic, DuplicateStringLiteral, LineLength */
pipeline {
agent {
docker {
@ -9,18 +10,14 @@ pipeline {
}
}
stages {
stage('Prepare Environment'){
steps{
sh "#!/bin/sh \n" +
stage('Prepare Environment') {
steps {
sh '#!/bin/sh \n' +
'id; apk add docker openrc git'
}
}
stage('Obtain Source'){
stage('Obtain Source') {
steps {
git branch: 'main', url: 'https://git.adamoutler.com/aoutler/aidgaf-server.git'
}
@ -28,42 +25,35 @@ pipeline {
stage('Build in docker') {
steps {
// Get some code from a Git repository
sh "#!/bin/sh \n" +
sh '#!/bin/sh \n' +
'docker build -t aidgaf .'
}
}
stage ("setup credentials"){
steps{
stage('setup credentials') {
steps {
withCredentials([ sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey', usernameVariable: 'user')]) {
sh "#!/bin/sh \n" +
sh '#!/bin/sh \n' +
'set +e; docker stop aidgaf-server||echo machine stopped; docker rm aidgaf-server||echo machine does not exist; set -e'
}
}
}
stage ('export docker container'){
stage('export docker container') {
steps {
sh "#!/bin/sh \n" +
sh '#!/bin/sh \n' +
'set +e; docker stop aidgaf-server||echo machine stopped; docker rm aidgaf-server||echo machine does not exist; set -e'
withCredentials([
string(credentialsId: 'OpenAI-API-Token', variable: 'OPEN_AI_TOKEN'),
string(credentialsId: 'PapaHashingSecret', variable: 'PAPA_HASH'),
sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey', usernameVariable: 'user')
string(credentialsId: 'PapaAsyncUrl', variable: 'ASYNC_URL'),
sshUserPrivateKey(credentialsId: 'dockeruserOn192.168.1.115', keyFileVariable: 'sshkey')
]) {
sh "#!/bin/sh \n" +
'mkdir -p ~/.ssh; cp "${sshkey}" ~/.ssh/id_rsa'
sh "#!/bin/sh \n" +
'docker run --name=aidgaf-server -eSERVERPORT=8087 -eHOSTNAME=0.0.0.0 -eHASHKEY="${PAPA_HASH}" -eAPIKEY="${OPEN_AI_TOKEN}" -p8087:8087 -d --restart=always aidgaf'
sh '#!/bin/sh \n' +
'mkdir -p ~/.ssh; cp "$sshkey" ~/.ssh/id_rsa'
sh '#!/bin/sh \n' +
/* groovylint-disable-next-line GStringExpressionWithinString */
'docker run --name=aidgaf-server -eSERVERPORT=8087 -eHOSTNAME=0.0.0.0 -eHASHKEY="${PAPA_HASH}" -eAPIKEY="${OPEN_AI_TOKEN}" -eASYNC_METHOD="PATCH" -eASYNC_URL="${} -p8087:8087 -d --restart=always aidgaf'
}
}
}
// To run Maven on a Windows agent, use
// bat "mvn -Dmaven.test.failure.ignore=true clean package"
}
}

7
src/aidgaf/aidgaf-server/__init__.py
View File

@ -1,7 +0,0 @@
""" aidgaf-server. """
""" The name of the package"""
name = 'aidgaf'
""" The version of the package"""
version='0.1.0'

5
src/aidgaf/aidgaf-server/__main__.py
View File

@ -1,5 +0,0 @@
""" aidgaf-server entry point. """
import server
""" The main entry point for the aidgaf-server package."""
server.main()

4
src/aidgaf/aidgaf-server/const.py
View File

@ -1,4 +0,0 @@
"""Constants for the aidgaf-server package."""
""" The encoding to use for strings. """
UTF8="utf-8"

7
src/aidgaf/aidgaf_server/__init__.py Normal file
View File

@ -0,0 +1,7 @@
""" aidgaf-server. """
# The name of the package
name = 'aidgaf_server' # pylint: disable=C0103
# The version of the package
version = '0.1.0' # pylint: disable=C0103

5
src/aidgaf/aidgaf_server/__main__.py Normal file
View File

@ -0,0 +1,5 @@
""" aidgaf-server entry point. """
import server
# The main entry point for the aidgaf-server package.
server.main()

10
src/aidgaf/aidgaf_server/const.py Normal file
View File

@ -0,0 +1,10 @@
"""Constants for the aidgaf-server package."""
UTF8="utf-8"
""" The encoding to use for strings. """
OPENAI_TIMEOUT=30
""" The timeout for OpenAI requests. """
REPORTING_TIMEOUT=10
""" The timeout for async reporting requests. """

10
src/aidgaf/aidgaf-server/hash_calculator.py → src/aidgaf/aidgaf_server/hash_calculator.py
View File

@ -1,6 +1,8 @@
""" This module contains the function to calculate the hash of a value and a secret."""
import hashlib
def calculate_hash(value:bytes, secret:bytes)->str:
def calculate_hash(value: bytes, secret: bytes) -> str:
""" This function calculates the hash of a value and a secret.
It is used to verify that the message is from the server. The hash
is calculated using the SHA512 algorithm. The hash is returned as a
@ -13,6 +15,6 @@ def calculate_hash(value:bytes, secret:bytes)->str:
returns:
The hash of the value and secret. This is a hex string.
"""
m = hashlib.sha512()
m.update(b"".join([value,secret]))
return m.hexdigest()
sha512 = hashlib.sha512()
sha512.update(b"".join([value, secret]))
return sha512.hexdigest()

60
src/aidgaf/aidgaf-server/idgaf.py → src/aidgaf/aidgaf_server/idgaf.py
View File

@ -1,61 +1,61 @@
""" This file contains the logic for the IDGAF server. It can be used to make a stand-alone
IDGAF server, or it can be used as a module in a larger application.
"""
from datetime import datetime
import json
import random
import security
import re
import requests
from time import time
from datetime import datetime
import settings
from const import OPENAI_TIMEOUT
""" The URL for the OpenAI API. """
URL = "https://api.openai.com/v1/completions"
""" The URL for the OpenAI API. """
""" The data to send to OpenAI. """
DATA = {"model": settings.OPEN_AI_COMPLETION_MODEL,
"prompt": settings.PROMPTS[0],
"temperature": settings.TEMPERATURE,
"max_tokens": settings.OPEN_AI_MAX_TOKENS
}
""" The data to send to OpenAI. """
""" The headers to send to OpenAI. """
request_headers = {"Authorization": "Bearer " +
settings.APIKEY, "Content-Type": "application/json"}
""" The headers to send to OpenAI. """
def get_response_base_object(text:str) -> dict:
def get_response_base_object(text: str) -> dict:
""" This is used to create the response object for the server.
Parameters:
text: The text to return to the client.
Returns:
A dictionary containing the response object.
"""
resultObject = {}
resultObject["message"] = {}
resultObject["service"] = "AIDGAF Server"
resultObject["message"]["data"] = {}
resultObject["message"]["data"]["resultObject"] = text
resultObject["timestamp"] = datetime.utcnow().timestamp()
return resultObject
result_object = {}
result_object["message"] = {}
result_object["service"] = "AIDGAF Server"
result_object["message"]["data"] = {}
result_object["message"]["data"]["resultObject"] = text
result_object["timestamp"] = datetime.utcnow().timestamp()
return result_object
def parse_idgaf_request(command)->[int, dict]:
def parse_idgaf_request(idgaf_command) -> [int, dict]:
""" This function handles the IDGAF command. It will return a response object.
Parameters:
command: The command object received from the client.
Returns:
A tuple containing the status code and the response object.
"""
the_data = get_prompt(command)
the_data = get_prompt(idgaf_command)
response = get_gpt_response(the_data)
try:
response_text = response.json()['choices'][0]['text'].strip()
except (KeyError):
response_text = gpt_response.text
except KeyError:
response_text = response.text
obj = get_response_base_object(response_text)
return [response.status_code, obj]
def get_gpt_response(data) -> requests.Response:
""" This function communicates with OpenAI and returns a response object.
Parameters:
@ -63,11 +63,14 @@ def get_gpt_response(data) -> requests.Response:
Returns:
The response object from OpenAI.
"""
gpt_response = requests.post(URL, json=data, headers=request_headers)
gpt_response = requests.post(
url=URL, json=data, headers=request_headers, timeout=OPENAI_TIMEOUT)
return gpt_response
def get_prompt(command)->dict:
""" Selects a prompt from the PROMPTS list and replaces the USERNAME placeholder with the username.
def get_prompt(command) -> dict:
""" Selects a prompt from the PROMPTS list and replaces the USERNAME placeholder
with the username.
Parameters:
command: The command object received from the client.
Returns:
@ -83,9 +86,14 @@ def get_prompt(command)->dict:
return the_data
# This function is for testing the IDGAF capabilities without a server.
if __name__ == "__main__":
""" This function is for testing the IDGAF capabilities without a server. """
value = '{"service":"papa","message":{"command":"aidgaf","data":{"username":"AdamOutler"},"timestamp":1675725191},"hash":"1bc73914478835d03f9ebdfb46328321d2bb656647e2876d6f162cc1860607fcfca8d825c48e390a6a254ee0835c8a4fe5f9a25795a3a0880ae5a23e9c132cf2"}'
command = json.loads(value)
[code, result] = parse_idgaf_request(command)
INPUT = '''{"service":"papa","message":
{"command":"aidgaf","data":{"username":"AdamOutler"},
"timestamp":1675725191},
"hash":"1bc73914478835d03f9ebdfb46328321d2bb656647e28
76d6f162cc1860607fcfca8d825c48e390a6a254ee0835c8a4fe5f
9a25795a3a0880ae5a23e9c132cf2"}'''
test_command = json.loads(INPUT)
[code, result] = parse_idgaf_request(test_command)
print(result)

12
src/aidgaf/aidgaf-server/merge.py → src/aidgaf/aidgaf_server/merge.py
View File

@ -1,18 +1,18 @@
"""This module contains the merge_dict function."""
from functools import reduce
def merge_dict_no_overwrite(base_dictionary, other_dictionary, location=None)->dict:
""" This function merges two dictionaries. If the same key exists in both dictionaries,
the value from the other_dictionary will be used. This function will recurse into nested
dictionaries. If the same key exists in both dictionaries, and the value is a dictionary,
the function will recurse into the nested dictionary. If the same key exists in both dictionaries,
and the value is not a dictionary, the value from the base_dictionary will be used.
the function will recurse into the nested dictionary. If the same key exists in both
dictionaries, and the value is not a dictionary, the value from the base_dictionary will
be used.
Parameters:
base_dictionary: The dictionary to merge the other_dictionary into.
other_dictionary: The dictionary to merge keys from. if the same key exists in both dictionaries,
the value from the base_dictionary will be used.
other_dictionary: The dictionary to merge keys from. if the same key exists in both
dictionaries, the value from the base_dictionary will be used.
location: leave blank. This is used to track the location of the key in the dictionary.
Returns:
The merged dictionary.
@ -29,5 +29,3 @@ def merge_dict_no_overwrite(base_dictionary, other_dictionary, location=None)->d
else:
base_dictionary[key] = other_dictionary[key]
return base_dictionary

8
src/aidgaf/aidgaf-server/security.py → src/aidgaf/aidgaf_server/security.py
View File

@ -13,9 +13,9 @@ def perform_hash_checks(str_request)->bool:
str_request: The request body, as a string.
Returns:
True if the message is valid, False otherwise."""
hash = get_message_hash(str_request)
my_hash = get_message_hash(str_request)
if hash not in str_request:
if my_hash not in str_request:
print("Error: hash not match")
return False
if not verify_message_time(str_request):
@ -32,12 +32,12 @@ def get_message_hash(json_command) -> str:
The hash of the message, as a string. """
strip1 = re.sub(".*\"message\":", "", json_command, 1)
if ("\"hash\":" in strip1):
if "\"hash\":" in strip1:
strip2 = re.sub(',\"hash\":.*', '', strip1)
else:
strip2 = strip1[:-1]
json_value = bytes(strip2, UTF8)
if (settings.HASHKEY is not None):
if settings.HASHKEY is not None:
hash_value = hash_calculator.calculate_hash(
json_value, settings.HASHKEY)
return hash_value

42
src/aidgaf/aidgaf-server/server.py → src/aidgaf/aidgaf_server/server.py
View File

@ -1,16 +1,15 @@
""" aidgaf-server - A simple server to handle requests from the aidgaf client.
This server will communicate with OpenAI to generate responses to the client."""
from http.server import BaseHTTPRequestHandler, HTTPServer
import json
import random
import threading
import requests
import merge
from http.server import BaseHTTPRequestHandler, HTTPServer
import merge
import idgaf
import settings
import security
from const import UTF8
from const import UTF8, REPORTING_TIMEOUT
default_request_body = b'{"message":{"command":"aidgaf","data":{"username":"AdamOutler"}}}'
request_headers = {}
@ -23,13 +22,18 @@ class IDGAFServer(BaseHTTPRequestHandler):
self.send_response(418)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(bytes("", UTF8))
self.wfile.write(bytes("HTTP 408\nI'm a teapot", UTF8))
self.close_connection
def do_PATCH(self):
""" This function handles PATCH requests. """
body = self.get_body().decode(UTF8)
if not perform_sanity_checks(body):
check_result=perform_sanity_checks(body)
if check_result:
self.send_response(403)
self.end_headers()
self.wfile.write(bytes(check_result, UTF8))
self.close_connection
return
command = json.loads(body)
self.do_request_handling(command)
@ -73,19 +77,18 @@ class IDGAFServer(BaseHTTPRequestHandler):
if (code != 200):
return
if (settings.ASYNC_METHOD == "POST"):
requests.post(settings.ASYNC_URL, data=data, headers=request_headers)
requests.post(settings.ASYNC_URL, data=data, headers=request_headers,
timeout=REPORTING_TIMEOUT)
elif (settings.ASYNC_METHOD == "PUT"):
requests.put(settings.ASYNC_URL, data=data, headers=request_headers)
requests.put(settings.ASYNC_URL, data=data, headers=request_headers,
timeout=REPORTING_TIMEOUT)
elif (settings.ASYNC_METHOD == "PATCH"):
resp=requests.patch(settings.ASYNC_URL, data=data, headers=request_headers)
resp=requests.patch(settings.ASYNC_URL, data=data, headers=request_headers,
timeout=REPORTING_TIMEOUT)
print(str(resp)+resp.text)
# gpt_response = requests.post(URL, json=data, headers=request_headers)
def getGPTResponse(self, command) -> [int, str]:
@ -124,9 +127,10 @@ class IDGAFServer(BaseHTTPRequestHandler):
self.end_headers()
print("sending:"+body)
self.wfile.write(bytes(body, UTF8))
self.close_connection
def perform_sanity_checks(str_request) -> bool:
def perform_sanity_checks(str_request) -> str:
""" Performs a hash check on the message, and verifies the timestamp is valid.
If either check fails, return False. Otherwise, return True.
Parameters:
@ -135,13 +139,13 @@ def perform_sanity_checks(str_request) -> bool:
True if the message is valid, False otherwise. """
if settings.HASHKEY is not None:
hash = security.get_message_hash(str_request)
if hash not in str_request:
if settings.HASHKEY and hash not in str_request:
print("Error: hash not match")
return False
if not security.verify_message_time(str_request):
return "Error: hash not match"
if settings.HASHKEY and not security.verify_message_time(str_request):
print("Error: timestamp expired")
return False
return True
return "Error: timestamp expired"
return ""
def main():

5
src/aidgaf/aidgaf-server/settings.py → src/aidgaf/aidgaf_server/settings.py
View File

@ -42,7 +42,8 @@ OPEN_AI_COMPLETION_MODEL = "text-davinci-003"
TEMPERATURE = 0.7
""" The hash key for the server. Leave this blank if you don't want to use it. """
HASHKEY = bytes(os.getenv('HASHKEY'),
UTF8) # shared secret for hmac of message
HASHKEY = bytes(os.getenv('HASHKEY') or "",UTF8) # shared secret for hmac of message
if (HASHKEY == ""):
HASHKEY=None
""" The maximum age of a message in seconds. Only used if HASHKEY is set."""
MAX_MESSAGE_AGE = 600