optimize app

.gitignore

@@ -6,4 +6,4 @@ zeroshot_onnx_dir/
 sent_clf_onnx_dir/
 zs_onnx_dir/
 sent_onnx_mdl_dir/
-sent_mdl_dir/
+sent_mdl_dir/

app.py

@@ -136,9 +136,9 @@ def sentiment_task_selected(task,
 
     #create inference session
     sentiment_session = ort.InferenceSession(f"{sent_onnx_mdl_dir}/{sent_onnx_mdl_name}")
-    sentiment_session_quant = ort.InferenceSession(f"{sent_onnx_mdl_dir}/{sent_onnx_quant_mdl_name}")
+    # sentiment_session_quant = ort.InferenceSession(f"{sent_onnx_mdl_dir}/{sent_onnx_quant_mdl_name}")
 
-    return model_sentiment,tokenizer_sentiment,sentiment_session,sentiment_session_quant
+    return model_sentiment,tokenizer_sentiment,sentiment_session
 
 ############## Pre-Download & instantiate objects for sentiment analysis ********************* END **********************************
 
@@ -167,35 +167,30 @@ def zs_task_selected(task,
 
     #create inference session from onnx model
     zs_session = ort.InferenceSession(f"{zs_onnx_mdl_dir}/{zs_onnx_mdl_name}")
-    zs_session_quant = ort.InferenceSession(f"{zs_onnx_mdl_dir}/{zs_onnx_quant_mdl_name}")
+    # zs_session_quant = ort.InferenceSession(f"{zs_onnx_mdl_dir}/{zs_onnx_quant_mdl_name}")
 
-    return tokenizer_zs,zs_session,zs_session_quant
+    return tokenizer_zs,zs_session
 
 ############## Pre-Download & instantiate objects for Zero shot analysis ********************* END **********************************
 
 
 if select_task == 'Detect Sentiment':
-
     t1=time.time()
     model_sentiment,tokenizer_sentiment,\
-    sentiment_session,sentiment_session_quant = sentiment_task_selected(task=select_task)
+    sentiment_session = sentiment_task_selected(task=select_task)
     t2 = time.time()
     st.write(f"Total time to load Model is {(t2-t1)*1000:.1f} ms")
 
     st.header("You are now performing Sentiment Analysis")
     input_texts = st.text_input(label="Input texts separated by comma")
-    c1,c2,c3,c4=st.columns(4)
+    c1,c2,_,_=st.columns(4)
 
     with c1:
         response1=st.button("Normal runtime")
     with c2:
         response2=st.button("ONNX runtime")
-    with c3:
-        response3=st.button("ONNX runtime with Quantization")
-    with c4:
-        response4 = st.button("Simulate 100 runs each runtime")
 
-    if any([response1,response2,response3,response4]):
+    if any([response1,response2]):
         if response1:
             start=time.time()
             sentiments = classify_sentiment(input_texts,
@@ -211,65 +206,6 @@ if select_task == 'Detect Sentiment':
                                                _tokenizer=tokenizer_sentiment)
             end = time.time()
             st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
-        elif response3:
-            start = time.time()
-            sentiments=classify_sentiment_onnx(input_texts,
-                                                     _session=sentiment_session_quant,
-                                                     _tokenizer=tokenizer_sentiment)
-            end = time.time()
-            st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
-        elif response4:
-            normal_runtime=[]
-            for i in range(100):
-                start=time.time()
-                sentiments = classify_sentiment(input_texts,
-                                                model=model_sentiment,
-                                                tokenizer=tokenizer_sentiment)
-                end=time.time()
-                t = (end - start) * 1000
-                normal_runtime.append(t)
-            normal_runtime=np.clip(normal_runtime,10,60)
-
-            onnx_runtime=[]
-            for i in range(100):
-                start=time.time()
-                sentiments = classify_sentiment_onnx(input_texts,
-                                               _session=sentiment_session,
-                                               _tokenizer=tokenizer_sentiment)
-                end=time.time()
-                t=(end-start)*1000
-                onnx_runtime.append(t)
-            onnx_runtime = np.clip(onnx_runtime, 0, 20)
-
-            onnx_runtime_quant=[]
-            for i in range(100):
-                start=time.time()
-                sentiments = classify_sentiment_onnx(input_texts,
-                                               _session=sentiment_session_quant,
-                                               _tokenizer=tokenizer_sentiment)
-                end=time.time()
-
-                t=(end-start)*1000
-                onnx_runtime_quant.append(t)
-            onnx_runtime_quant = np.clip(onnx_runtime_quant, 0, 20)
-
-
-            temp_df=pd.DataFrame({'Normal Runtime (ms)':normal_runtime,
-                                  'ONNX Runtime (ms)':onnx_runtime,
-                                  'ONNX Quant Runtime (ms)':onnx_runtime_quant})
-
-            from plotly.subplots import make_subplots
-            fig = make_subplots(rows=1, cols=3, start_cell="bottom-left",
-                                subplot_titles=['Normal Runtime','ONNX Runtime','ONNX Runtime with Quantization'])
-
-            fig.add_trace(go.Histogram(x=temp_df['Normal Runtime (ms)']),row=1,col=1)
-            fig.add_trace(go.Histogram(x=temp_df['ONNX Runtime (ms)']),row=1,col=2)
-            fig.add_trace(go.Histogram(x=temp_df['ONNX Quant Runtime (ms)']),row=1,col=3)
-            fig.update_layout(height=400, width=1000,
-                              title_text="100 Simulations of different Runtimes",
-                              showlegend=False)
-            st.plotly_chart(fig,config=_plotly_config )
-
         else:
             pass
         for i,t in enumerate(input_texts.split(',')):
@@ -281,9 +217,8 @@ if select_task == 'Detect Sentiment':
                                          color_background='rgb(233, 116, 81)',key=t)
 
 if select_task=='Zero Shot Classification':
-
     t1=time.time()
-    tokenizer_zs,zs_session,zs_session_quant = zs_task_selected(task=select_task)
+    tokenizer_zs,zs_session = zs_task_selected(task=select_task)
     t2 = time.time()
     st.write(f"Total time to load Model is {(t2-t1)*1000:.1f} ms")
 
@@ -291,46 +226,25 @@ if select_task=='Zero Shot Classification':
     input_texts = st.text_input(label="Input text to classify into topics")
     input_lables = st.text_input(label="Enter labels separated by commas")
 
-    c1,c2,c3,c4=st.columns(4)
+    c1,_,_,_=st.columns(4)
 
     with c1:
-        response1=st.button("ONNX runtime")
-    with c2:
-        response2=st.button("ONNX runtime Quantized")
-    # with c3:
-    #     response3=st.button("ONNX runtime with Quantization")
-    # with c4:
-    #     response4 = st.button("Simulate 10 runs each runtime")
-
-    if any([response1,response2]):
-        if response1:
-            start = time.time()
-            df_output=zero_shot_classification_onnx(premise=input_texts,labels=input_lables,_session=zs_session,_tokenizer=tokenizer_zs)
-            end=time.time()
-            st.write("")
-            st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
-
-            fig = px.bar(x='Probability',
-                         y='labels',
-                         text='Probability',
-                         data_frame=df_output,
-                         title='Zero Shot Normalized Probabilities')
-
-            st.plotly_chart(fig,config=_plotly_config)
-        elif response2:
-            start = time.time()
-            df_output = zero_shot_classification_onnx(premise=input_texts, labels=input_lables, _session=zs_session_quant,
-                                                   _tokenizer=tokenizer_zs)
-            end = time.time()
-            st.write("")
-            st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
-            fig = px.bar(x='Probability',
-                         y='labels',
-                         text='Probability',
-                         data_frame=df_output,
-                         title='Zero Shot Normalized Probabilities')
-
-            st.plotly_chart(fig, config=_plotly_config)
-        else:
-            pass
+        response1=st.button("Compute with ONNX runtime")
+
+    if response1:
+        start = time.time()
+        df_output = zero_shot_classification_onnx(premise=input_texts, labels=input_lables, _session=zs_session,
+                                               _tokenizer=tokenizer_zs)
+        end = time.time()
+        st.write("")
+        st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
+        fig = px.bar(x='Probability',
+                     y='labels',
+                     text='Probability',
+                     data_frame=df_output,
+                     title='Zero Shot Normalized Probabilities')
+
+        st.plotly_chart(fig, config=_plotly_config)
+    else:
+        pass
 

sentiment_clf_helper.py

@@ -61,9 +61,9 @@ def create_onnx_model_sentiment(_model, _tokenizer,sent_onnx_mdl_dir=sent_onnx_m
                                      use_external_format=False
                                      )
 
-        quantize_dynamic(f"{sent_onnx_mdl_dir}/{sent_onnx_mdl_name}",
-                         f"{sent_onnx_mdl_dir}/{sent_onnx_quant_mdl_name}",
-                         weight_type=QuantType.QUInt8)
+        # quantize_dynamic(f"{sent_onnx_mdl_dir}/{sent_onnx_mdl_name}",
+        #                  f"{sent_onnx_mdl_dir}/{sent_onnx_quant_mdl_name}",
+        #                  weight_type=QuantType.QUInt8)
     else:
         pass
 

zeroshot_clf_helper.py

@@ -65,15 +65,13 @@ def create_onnx_model_zs(zs_onnx_mdl_dir=zs_onnx_mdl_dir):
         except Exception as e:
             print(e)
 
-        #create quanitzed model from vanila onnx
-        quantize_dynamic(f"{zs_onnx_mdl_dir}/{zs_onnx_mdl_name}",
-                         f"{zs_onnx_mdl_dir}/{zs_onnx_quant_mdl_name}",
-                         weight_type=QuantType.QUInt8)
+        # #create quanitzed model from vanila onnx
+        # quantize_dynamic(f"{zs_onnx_mdl_dir}/{zs_onnx_mdl_name}",
+        #                  f"{zs_onnx_mdl_dir}/{zs_onnx_quant_mdl_name}",
+        #                  weight_type=QuantType.QUInt8)
     else:
         pass
 
-create_onnx_model_zs(zs_onnx_mdl_dir=zs_onnx_mdl_dir)
-
 def zero_shot_classification_onnx(premise,labels,_session,_tokenizer):
     try:
         labels=labels.split(',')