api + refactors + config

api.py

@@ -0,0 +1,11 @@
+import requests
+from config import API_URL
+
+def define_en_get(word):
+    resp = requests.get(f"{API_URL}/define/en/{word}")
+    if resp:
+        print(resp.json())
+
+
+if __name__ == "__main__":
+    define_en_get("hello")

config.py

@@ -0,0 +1,11 @@
+
+INTENTS_FILEPATH = "./data/intents.json"
+
+MODEL_FILEPATH = "./out/intents.pth"
+MODEL_DEVICE = "cpu"
+
+# model
+CONFIDENCE_THRESHOLD = 0.75  # threshold where intent is accepted
+
+# api
+API_URL = "http://localhost:8000"

main.py

@@ -4,25 +4,24 @@
 
 from nn import NeuralNet
 import preprocess
+import config
 
 if len(sys.argv) != 2:
     raise ValueError("pass in an utterance")
 
 query = sys.argv[1]
 
 # load model data and rebuild neural net
-model_filepath = "./out/intents.pth"
-model_data = torch.load(model_filepath)
+model_data = torch.load(config.MODEL_FILEPATH)
 
 model_state = model_data["model_state"]
 input_size = model_data["input_size"]
 hidden_size = model_data["hidden_size"]
 output_size = model_data["output_size"]
 word_dict = model_data["word_dict"]
 tags = model_data["tags"]
-device = "cpu"
 
-model = NeuralNet(input_size, hidden_size, output_size).to(device)
+model = NeuralNet(input_size, hidden_size, output_size).to(config.MODEL_DEVICE)
 model.load_state_dict(model_state)
 model.eval()
 
@@ -47,8 +46,7 @@ def preprocess_query(query):
 probs = torch.softmax(output, dim=1)
 prob = probs[0][predicted.item()]
 
-confidence_threshold = 0.75
-if prob.item() > confidence_threshold:
+if prob.item() > config.CONFIDENCE_THRESHOLD:
     print(f"[prob={prob.item():.4f}] {tag}")
 else:
     print("query not understood")

reader.py

@@ -2,6 +2,8 @@
 import json
 from pipeop import pipes
 
+import config
+
 
 def import_intents(filepath):
     with open(filepath, "r") as filehandle:

requirements.txt

@@ -1,9 +1,16 @@
+certifi==2022.5.18.1
+charset-normalizer==2.0.12
 click==8.1.3
+idna==3.3
+importlib-metadata==4.11.4
 joblib==1.1.0
 nltk==3.7
-numpy==1.22.4
+numpy==1.21.6
 pipeop==0.3.0
 regex==2022.6.2
+requests==2.27.1
 torch==1.11.0
 tqdm==4.64.0
 typing_extensions==4.2.0
+urllib3==1.26.9
+zipp==3.8.0

train.py

@@ -9,13 +9,13 @@
 from dataset import IntentDataset
 from nn import NeuralNet
 from reader import import_intents, get_tags
+import config
 
 
 @pipes
-def train():
-
+def train(batch_size=8, num_workers=2, hidden_size=8, learning_rate=0.001, training_epochs=1000):
     # load data
-    intents = import_intents("./data/intents.json")
+    intents = import_intents(config.INTENTS_FILEPATH)
     tags = get_tags(intents)
 
     word_dict = []
@@ -49,8 +49,6 @@ def train():
     dataset = IntentDataset(x_data, y_data)
 
     # build dataloader
-    batch_size = 8
-    num_workers = 2
     loader = DataLoader(
         dataset=dataset,
         batch_size=batch_size,
@@ -60,21 +58,18 @@ def train():
 
     # build neural net
     input_size = len(word_dict)
-    hidden_size = 8
+
     output_size = len(tags)
-    device = "cpu"
-    model = NeuralNet(input_size, hidden_size, output_size).to(device)
+    model = NeuralNet(input_size, hidden_size, output_size).to(config.MODEL_DEVICE)
 
     # start training
-    learning_rate = 0.001
-    training_epochs = 1000
     criterion = nn.CrossEntropyLoss()
     optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
 
     for epoch in range(training_epochs):
         for (words, labels) in loader:
-            words = words.to(device)
-            labels = labels.to(device)
+            words = words.to(config.MODEL_DEVICE)
+            labels = labels.to(config.MODEL_DEVICE)
 
             # forward pass
             outputs = model(words)
@@ -91,7 +86,6 @@ def train():
     print(f"final loss={loss.item():.4f}")
 
     # save training data to file
-    model_filepath = "./out/intents.pth"
     model_data = {
         "model_state": model.state_dict(),
         "input_size": input_size,
@@ -100,7 +94,7 @@ def train():
         "word_dict": word_dict,
         "tags": tags
     }
-    torch.save(model_data, model_filepath)
+    torch.save(model_data, config.MODEL_FILEPATH)
     print("successfully saved model data to file")