Add more test sequences, scatter plot visualization

Author: mtszkw

Pipfile

@@ -6,6 +6,7 @@ name = "pypi"
 [packages]
 transformers = {extras = ["torch"], version = "*"}
 scipy = "*"
+matplotlib = "*"
 
 [dev-packages]
 

run_sentiment_classifier.py

@@ -1,3 +1,5 @@
+import numpy as np
+import matplotlib.pyplot as plt
 # https://stackoverflow.com/questions/7370801/how-to-measure-elapsed-time-in-python
 from timeit import default_timer as timer
 
@@ -8,6 +10,12 @@
 from utils import preprocess, download_label_mapping, output_vector_to_labels
 
 
+def read_test_sequences(path: str):
+    with open(path, 'r') as f:
+        sequences = [x.rstrip() for x in f.readlines()]
+        return sequences
+
+
 def run_model(model, tokenized_input):
     output = model(**tokenized_input)
     return output_vector_to_labels(output, download_label_mapping())
@@ -19,37 +27,62 @@ def check_inference_time(model, tokenized_input):
     elapsed_time = timer()-t
     return elapsed_time
 
+
 if __name__ == "__main__":
     tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment", torchscript=True)
     clf = AutoModelForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment", torchscript=True)
 
-    input_texts = [
-        "Hello world",
-        "Happy birthday",
-        "I don't think it's gonna work",
-        "I enjoy natural language understanding"
-    ]
+    n_experiments = 5
+    input_texts = [preprocess(x) for x in read_test_sequences("test_sequences.txt")]
 
-    # 1. Vanilla
+    # 1. Eager
+    eager_measurements = np.zeros((n_experiments, len(input_texts)))
     tokenized_inputs = [tokenizer(x, return_tensors='pt') for x in input_texts]
-    outputs = [run_model(clf, x) for x in tokenized_inputs]
-    output_times = [check_inference_time(clf, x) for x in tokenized_inputs]
-    
-    for inp, out in zip(input_texts, outputs):
-        print(inp, out)
-
-    print(output_times)
 
-    print("")
+    for i in range(n_experiments):
+        # outputs = [run_model(clf, x) for x in tokenized_inputs]
+        eager_measurements[i] = [check_inference_time(clf, x) for x in tokenized_inputs]
+        # for inp, out in zip(input_texts, outputs):
+            # print(inp, '\n', out, '\n')
+        # print(output_times)
 
+    
     # 2. TorchScript (JIT)
+    script_measurements = np.zeros((n_experiments, len(input_texts)))
     tokenized_inputs = [tokenizer(x, return_tensors='pt') for x in input_texts]
     traced_model = torch.jit.trace(clf, (tokenized_inputs[0]['input_ids'], tokenized_inputs[0]['attention_mask']))
-    outputs = [run_model(traced_model, x) for x in tokenized_inputs]
-    output_times = [check_inference_time(traced_model, x) for x in tokenized_inputs]
     # torch.jit.save(traced_model, "traced_twitter_roberta_base_sentiment.pt")
     # loaded_model = torch.jit.load("traced_twitter_roberta_base_sentiment.pt")
-    for inp, out in zip(input_texts, outputs):
-        print(inp, out)
 
-    print(output_times)
+    for i in range(n_experiments):
+        # outputs = [run_model(traced_model, x) for x in tokenized_inputs]
+        script_measurements[i] = [check_inference_time(traced_model, x) for x in tokenized_inputs]
+        # for inp, out in zip(input_texts, outputs):
+            # print(inp, '\n', out, '\n')
+        # print(output_times)
+
+    print(eager_measurements)
+    print(script_measurements)
+
+    # Box Plot
+    
+    eager_avgs = np.mean(eager_measurements, axis=0)
+    script_avgs = np.mean(script_measurements, axis=0)
+    print(eager_avgs)
+    print(script_avgs)
+
+    # Scatter Plot
+
+    indices = np.tile(np.arange(len(input_texts)), n_experiments)
+    eager_measurements = eager_measurements.flatten()
+    script_measurements = script_measurements.flatten()
+    print(indices)
+    print(eager_measurements)
+
+    plt.style.use('seaborn')
+    plt.scatter(indices, eager_measurements, label='Eager mode')
+    plt.scatter(indices, script_measurements, label='Script mode')
+    plt.xlabel('Sequence ID')
+    plt.ylabel('Inference time [s]')
+    plt.legend()
+    plt.show()

test_sequences.txt

@@ -0,0 +1,14 @@
+Happy birthday
+Taming transformers is a fun paper to read.
+Really happy to see this. Congratulations John :)
+Honored to be in such good company for the Bay Area’s Best Places to Work 2016
+I've always been inspired by how my friend @ramsri_goutham applies NLP to creative real-world use cases.
+At @gridai_ & @pytorchlightnin we strongly believe #AI should be #inclusive and celebrate #diversity
+We're excited to launch the third Habitat Challenge at the Embodied AI workshop with 15 research & academic institutions.
+I don't think it's gonna work
+Sometimes, You gotta hate #Windows updates. 
+Friend requests on @facebook I’m still confused
+@facebook… for f***s sake… Can’t find a post posted 3mins ago due to your stupid “non chronological” timeline
+Netflix's Chaos Monkey tool, but instead of randomly killing containers it randomly cancels meetings on my calendar
+All of these things are true: 1. Tech is terrible at hiring and interviewing. 2. Tech is terrible at defining different areas of data work.
+I’m so frustrated that many privileged people are ignoring racial/ethnic injustices of Covid-19 deaths. Widespread apathy is making these grave disparities more severe in younger adults. Generational damage is destroying families.