Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feature/UI'
Browse files Browse the repository at this point in the history
  • Loading branch information
MasterYip committed Oct 8, 2023
2 parents f8d5653 + 2b7fee2 commit b0bc7c0
Show file tree
Hide file tree
Showing 16 changed files with 2,193 additions and 375 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
*.py[cdi]
private_config.py

config.json
/release
####
PDFexample/*
!PDFexample/2016_ANYmal - a highly mobile and dynamic quadrupedal robot.pdf
Expand Down
9 changes: 7 additions & 2 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,14 @@ pymupdf = "*"
opencv-python = "*"
tqdm = "*"
openai = "*"
pillow = "*"
matplotlib = "*"
typing-extensions = "*"
tiktoken = "*"
pyqt5 = "*"

[dev-packages]

[requires]
python_version = "3.9"
python_full_version = "3.9.7"
python_version = "3.8"
python_full_version = "3.8.5"
1,232 changes: 905 additions & 327 deletions Pipfile.lock

Large diffs are not rendered by default.

Empty file removed __init__.py
Empty file.
5 changes: 3 additions & 2 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
APIBASE = "" # OpenAI API base, default is "https://api.openai.com/v1" for now (Leave it as empty if you are not sure)
APIKEYS = [""] # Your OpenAI API keys
MODEL = "gpt-3.5-turbo" # GPT model name
MAXTOKEN = 4096 # Max token number for each request (Cutoff is performed when the number of tokens exceeds this value)
MAXTOKEN = 4000 # Max token number for each request (Cutoff is performed when the number of tokens exceeds this value)
LANGUAGE = "English" # Only partially support Chinese
KEYWORD = "Science&Engineering" # Keyword for GPT model (What field you want the model to focus on)
PROXY = None # Your proxy address
Expand All @@ -28,7 +28,7 @@
THREAD_RATE_LIMIT = 100 # Each APIKEY can send 3 requests per minute (limited by OpenAI)
else: # Use fake GPT model
GPT_ENABLE = False
THREAD_RATE_LIMIT = 6000
THREAD_RATE_LIMIT = 6000


"""PDF Parser - Regular Expression"""
Expand All @@ -49,5 +49,6 @@
"""Xmind Sytle Template"""
TEMPLATE_XMIND_PATH = 'template.xmind'


"""Debuging"""
DEBUG_MODE = False
32 changes: 20 additions & 12 deletions gpt_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from threading import Thread
import time
from tqdm import trange
from config import *
import tiktoken


Expand All @@ -19,7 +18,7 @@ def countTokens(string: str, encoding_name: str) -> int:
return num_tokens


def autoCutoff(string: str, encoding_name: str = MODEL, max_tokens: int = MAXTOKEN) -> str:
def autoCutoff(string: str, encoding_name: str, max_tokens: int) -> str:
"""
Automatically cut off a string to a maximum number of tokens, using the given encoding.
:param string: The string to cut off.
Expand All @@ -34,7 +33,7 @@ def autoCutoff(string: str, encoding_name: str = MODEL, max_tokens: int = MAXTOK
return string


def messageAutoCutoff(message: list, encoding_name: str = MODEL, max_tokens: int = MAXTOKEN):
def messageAutoCutoff(message: list, encoding_name: str, max_tokens: int):
token_cnt = 0
for item in message:
token_cnt += countTokens(item['content'], encoding_name)
Expand All @@ -49,7 +48,8 @@ def messageAutoCutoff(message: list, encoding_name: str = MODEL, max_tokens: int

class GPTRequest(object):
# TODO: other model support?
def __init__(self, content, model=MODEL, language=LANGUAGE, keyword=KEYWORD):
def __init__(self, content, model=MODEL, language=LANGUAGE, keyword=KEYWORD, maxtoken=MAXTOKEN,
gpt_enable=GPT_ENABLE, fake_response=FAKE_GPT_RESPONSE):
"""
Initialize GPTRequest.\\
Note that request message has not been generated yet.
Expand All @@ -63,6 +63,10 @@ def __init__(self, content, model=MODEL, language=LANGUAGE, keyword=KEYWORD):
self.content = content
self.language = language
self.keyword = keyword
self.maxtoken = maxtoken
self.gpt_enable = gpt_enable
self.fake_response = fake_response

self.ret = None
self.success = False
# TODO: Nontype problem?
Expand Down Expand Up @@ -166,12 +170,12 @@ def run(self, errsleep=20):
FIXME: this sleep is not controlled by the GPTThread
"""
completions = None
if GPT_ENABLE:
if self.gpt_enable:
while completions is None:
try:
completions = openai.ChatCompletion.create(
model=self.model,
messages=messageAutoCutoff(self.message),
messages=messageAutoCutoff(self.message, self.model, self.maxtoken),
)
except Exception as e:
print(e)
Expand All @@ -185,7 +189,7 @@ def run(self, errsleep=20):
self.ret = completions['choices'][0]['message']['content']
self.postprocess()
else:
self.ret = FAKE_GPT_RESPONSE
self.ret = self.fake_response
self.postprocess()
self.success = True

Expand All @@ -195,10 +199,14 @@ def isSuccess(self):

class GPTThread(object):

def __init__(self, apikey, rate_limit=THREAD_RATE_LIMIT):
def __init__(self, apikey, rate_limit, gpt_enable):
self.apikey = apikey
self.thread = None
self.cooling_time = 60 / rate_limit
# FIXME: Should not use global variable
if gpt_enable:
self.cooling_time = 60 / rate_limit
else:
self.cooling_time = 0
self.last_request_time = None
self.thread = None

Expand All @@ -223,8 +231,8 @@ def isAvailable(self):


class GPTInterface(object):
def __init__(self, apibase=APIBASE, apikeys=APIKEYS, model=MODEL, keyword=KEYWORD,
gpt_enable=GPT_ENABLE, openai_proxy=PROXY) -> None:
def __init__(self, apibase, apikeys, model, keyword,
gpt_enable, openai_proxy, rate_limit) -> None:
if openai_proxy:
openai.proxy = openai_proxy
if apibase:
Expand All @@ -234,7 +242,7 @@ def __init__(self, apibase=APIBASE, apikeys=APIKEYS, model=MODEL, keyword=KEYWOR
self.keyword = keyword
self.gpt_enable = gpt_enable
self.request_list = []
self.thread_list = [GPTThread(apikey) for apikey in self.apikeys]
self.thread_list = [GPTThread(apikey, rate_limit, gpt_enable) for apikey in self.apikeys]
self.request_index = {}

def gptMultitask(self, interval=0):
Expand Down
Binary file added icon.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
208 changes: 208 additions & 0 deletions main_window.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
'''
Author: MasterYip 2205929492@qq.com
Date: 2023-10-08 09:23:35
LastEditors: MasterYip
LastEditTime: 2023-10-08 17:33:32
FilePath: \ChatPaper2Xmind\main_window.py
Description: file content
'''

import os
import re
import sys
import json
from threading import Thread
from config import *
from paper2xmind import pdf_batch_processing
from user_interface_ui_pyqt5 import *
from PyQt5.QtCore import QThread, pyqtSignal, QRect
from PyQt5.QtGui import QTextCursor, QIcon
from PyQt5.QtWidgets import QApplication, QMainWindow, QLineEdit

# Directory Management
try:
# Run in Terminal
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
except:
# Run in ipykernel & interactive
ROOT_DIR = os.getcwd()


class OutputRedirectThread(QThread):
signalForText = pyqtSignal(str)

def __init__(self, data=None, parent=None):
super(OutputRedirectThread, self).__init__(parent)
self.data = data

def write(self, text):
self.signalForText.emit(str(text))


class FileEdit(QLineEdit):
def __init__(self, parent):
super(FileEdit, self).__init__(parent)

self.setDragEnabled(True)

def dragEnterEvent(self, event):
data = event.mimeData()
urls = data.urls()
if (urls and urls[0].scheme() == 'file'):
event.acceptProposedAction()

def dragMoveEvent(self, event):
data = event.mimeData()
urls = data.urls()
if (urls and urls[0].scheme() == 'file'):
event.acceptProposedAction()

def dropEvent(self, event):
data = event.mimeData()
urls = data.urls()
if (urls and urls[0].scheme() == 'file'):
# for some reason, this doubles up the intro slash
filepath = str(urls[0].path())[1:]
self.setText(filepath)


class MainWindow(QMainWindow, Ui_MainWindow):
textColors = ['FF5C5C', '398AD9', '5BEC8D',
'FD42AC', 'FF33FF', '4B8200', 'DE87B8']
config_dir = os.path.join(ROOT_DIR, "config.json")

def __init__(self, parent=None):
super().__init__()
self.setupUi(self)
# Output Redirection
self.th = OutputRedirectThread()
self.th.signalForText.connect(self.onUpdateTerminal)
sys.stdout = self.th
sys.stderr = self.th
self.config = {}

self.setWindowIcon(QIcon(os.path.join(ROOT_DIR, "icon.png")))

# Default config
self.comboBox_LANGUAGE.addItems(["English", "Chinese"])
self.comboBox_MODEL.addItems(["gpt-3.5-turbo"])

self.lineEdit_APIBASE.setText(APIBASE)
self.textEdit_APIKEYS.setText("\n".join(APIKEYS))
self.comboBox_LANGUAGE.setCurrentText(LANGUAGE)
self.comboBox_MODEL.setCurrentText(MODEL)
self.spinBox_MAXTOKEN.setValue(MAXTOKEN)
self.lineEdit_KEYWORD.setText(KEYWORD)
self.lineEdit_PROXY.setText(PROXY)

self.checkBox_GEN_IMGS.setChecked(GEN_IMGS)
self.checkBox_GEN_EQUATIONS.setChecked(GEN_EQUATIONS)
self.checkBox_USE_PDFFIGURE2.setChecked(USE_PDFFIGURE2)
self.checkBox_SNAP_WITH_CAPTION.setChecked(SNAP_WITH_CAPTION)
self.lineEdit_FAKE_GPT_RESPONSE.setText(FAKE_GPT_RESPONSE)
self.checkBox_GPT_ENABLE.setChecked(GPT_ENABLE)
self.spinBox_TEXT2LIST_MAX_NUM.setValue(TEXT2LIST_MAX_NUM)
self.spinBox_TEXT2TREE_MAX_NUM.setValue(TEXT2TREE_MAX_NUM)
self.spinBox_THREAD_RATE_LIMIT.setValue(THREAD_RATE_LIMIT)

self.load_config()

# Path drag and drop
self.lineEdit_PATH = FileEdit(self.groupBox_3)
self.lineEdit_PATH.setObjectName(u"lineEdit_PATH")
self.lineEdit_PATH.setGeometry(QRect(20, 10, 311, 20))

self.pushButton_GENERATE.clicked.connect(self.generate_xmind)
self.pushButton_SAVE_CONFIG.clicked.connect(self.save_config)

def load_config(self):
if os.path.isfile(self.config_dir):
with open(self.config_dir, 'r') as f:
config = json.load(f)
self.lineEdit_APIBASE.setText(config['APIBASE'])
self.textEdit_APIKEYS.setText("\n".join(config['APIKEYS']))
self.comboBox_LANGUAGE.setCurrentText(config['LANGUAGE'])
self.comboBox_MODEL.setCurrentText(config['MODEL'])
self.spinBox_MAXTOKEN.setValue(config['MAXTOKEN'])
self.lineEdit_KEYWORD.setText(config['KEYWORD'])
self.lineEdit_PROXY.setText(config['PROXY'])

self.checkBox_GEN_IMGS.setChecked(config['GEN_IMGS'])
self.checkBox_GEN_EQUATIONS.setChecked(config['GEN_EQUATIONS'])
self.checkBox_USE_PDFFIGURE2.setChecked(config['USE_PDFFIGURE2'])
self.checkBox_SNAP_WITH_CAPTION.setChecked(
config['SNAP_WITH_CAPTION'])
self.lineEdit_FAKE_GPT_RESPONSE.setText(
config['FAKE_GPT_RESPONSE'])
self.checkBox_GPT_ENABLE.setChecked(config['GPT_ENABLE'])
self.spinBox_TEXT2LIST_MAX_NUM.setValue(
config['TEXT2LIST_MAX_NUM'])
self.spinBox_TEXT2TREE_MAX_NUM.setValue(
config['TEXT2TREE_MAX_NUM'])
self.spinBox_THREAD_RATE_LIMIT.setValue(
config['THREAD_RATE_LIMIT'])
self.config = config

def update_config(self):
self.config['APIBASE'] = self.lineEdit_APIBASE.text()
self.config['APIKEYS'] = self.textEdit_APIKEYS.toPlainText().splitlines()
self.config['LANGUAGE'] = self.comboBox_LANGUAGE.currentText()
self.config['MODEL'] = self.comboBox_MODEL.currentText()
self.config['MAXTOKEN'] = self.spinBox_MAXTOKEN.value()
self.config['KEYWORD'] = self.lineEdit_KEYWORD.text()
self.config['PROXY'] = self.lineEdit_PROXY.text()

self.config['GEN_IMGS'] = self.checkBox_GEN_IMGS.isChecked()
self.config['GEN_EQUATIONS'] = self.checkBox_GEN_EQUATIONS.isChecked()
self.config['USE_PDFFIGURE2'] = self.checkBox_USE_PDFFIGURE2.isChecked()
self.config['SNAP_WITH_CAPTION'] = self.checkBox_SNAP_WITH_CAPTION.isChecked()
self.config['FAKE_GPT_RESPONSE'] = self.lineEdit_FAKE_GPT_RESPONSE.text()
self.config['GPT_ENABLE'] = self.checkBox_GPT_ENABLE.isChecked()
self.config['TEXT2LIST_MAX_NUM'] = self.spinBox_TEXT2LIST_MAX_NUM.value()
self.config['TEXT2TREE_MAX_NUM'] = self.spinBox_TEXT2TREE_MAX_NUM.value()
self.config['THREAD_RATE_LIMIT'] = self.spinBox_THREAD_RATE_LIMIT.value()

def save_config(self):
self.update_config()
with open(self.config_dir, 'w') as f:
json.dump(self.config, f, indent=4)

def generate_xmind(self):
self.update_config()
process_thread = Thread(target=self.generate_xmind_thread)
process_thread.start()

def generate_xmind_thread(self):
pdf_batch_processing(self.lineEdit_PATH.text(), usePDFFigure2=self.config['USE_PDFFIGURE2'],
apibase=self.config['APIBASE'], apikeys=self.config['APIKEYS'],
model=self.config['MODEL'], keyword=self.config['KEYWORD'],
gpt_enable=self.config['GPT_ENABLE'], openai_proxy=self.config['PROXY'],
rate_limit=self.config['THREAD_RATE_LIMIT'])

def onUpdateTerminal(self, text):
self.textBrowser_Terminal.moveCursor(QTextCursor.End)
text = re.sub(r'\033\[\d+m', '', text)
self.textBrowser_Terminal.insertPlainText(text)
# FIXME: Insert colored text
# for line in text.splitlines():
# if line.startswith("\033[91m"):
# self.textBrowser_Terminal.setTextColor(
# QColor(self.textColors[0]))
# self.textBrowser_Terminal.insertPlainText(line[5:])
# elif line.startswith("\033[92m"):
# self.textBrowser_Terminal.setTextColor(
# QColor(self.textColors[1]))
# self.textBrowser_Terminal.insertPlainText(line[5:])
# else:
# # self.textBrowser_Terminal.setTextColor(
# # QColor(self.textColors[2]))
# self.textBrowser_Terminal.insertPlainText(line)
self.textBrowser_Terminal.moveCursor(QTextCursor.End)


if __name__ == '__main__':
app = QApplication(sys.argv)

myWin = MainWindow()
myWin.show()
app.exec()
Loading

0 comments on commit b0bc7c0

Please sign in to comment.