Upload 65 files

.gitattributes

@@ -35,3 +35,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 assets/AIM-CU-Overview.drawio.png filter=lfs diff=lfs merge=lfs -text
 docs/build/latex/aim-cu.pdf filter=lfs diff=lfs merge=lfs -text
+assets/AIM-CU-Overview.png filter=lfs diff=lfs merge=lfs -text
+assets/ui.png filter=lfs diff=lfs merge=lfs -text
+figure/fig_plot_average_metric.png filter=lfs diff=lfs merge=lfs -text
+figure/fig_plot_cusum_chart.png filter=lfs diff=lfs merge=lfs -text
+figure/fig_plot_data_distribution.png filter=lfs diff=lfs merge=lfs -text
+figure/fig_table_h_k_arl1.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -22,6 +22,7 @@ RUN Rscript -e "install.packages('spc')"
 WORKDIR /app/src/package
 COPY . /app/
 
-RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+RUN pip install --use-pep517 --no-cache-dir --upgrade -r /app/requirements.txt
+RUN pip install --upgrade gradio gradio_client
 
 CMD ["python3", "app.py"]

README.rst

@@ -1,20 +1,40 @@
-AIM-CU : A CUSUM-based tool for AI Monitoring
+AIM-CU: A CUSUM-based tool for AI Monitoring
 ======
 
-.. image:: assets/AIM-CU-Overview.drawio.png
+.. image:: assets/AIM-CU-Overview.png
     :width: 800
     :align: center
     :alt: AIM-CU flowchart
 
-Monitoring a clinically deployed AI device to detect performance drift is an essential step to
-ensure the safety and effectiveness of AI. 
+Monitoring a clinically deployed AI device to detect performance drift is an essential step to ensure the safety and effectiveness of AI. 
+
+AIM-CU is a statistical tool for AI monitoring based on a cumulative sum (AIM-CU) approach.
 
-AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU). 
 AIM-CU computes:
 
 * The parameter choices for change-point detection based on an acceptable false alarm rate
 * Detection delay estimates for a given displacement of the performance metric from the target for those parameter choices.
 
+System setup
+------------
+Make sure R is installed in the system. There is no specific version that this relies on. Here we have used R version 4.1.2 (2021-11-01). 
+Instructions for Linux (the below setup is only performed in Linux):
+
+.. code-block:: shell
+
+    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc |  tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
+    add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
+    apt-get install -y --no-install-recommends r-base r-base-dev
+
+    # setup R configs
+    echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.us.r-project.org'; options(repos = r);" > ~/.Rprofile
+    Rscript -e "install.packages('ggplot2')"
+    Rscript -e "install.packages('hexbin')"
+    Rscript -e "install.packages('lazyeval')"
+    Rscript -e "install.packages('cusumcharter')"
+    Rscript -e "install.packages('RcppCNPy')"
+    Rscript -e "install.packages('spc')"
+
 Code execution
 --------------
 Clone AIM-CU repository.
@@ -36,17 +56,30 @@ Run AIM-CU.
 
 .. code-block:: shell
 
+    cd src/package
     python3 app.py
 
 Open the URL http://0.0.0.0:7860 that is running the AIM-CU locally.
 
 Example code execution
 ----------------------
-Example code can be run in a Jupyter Notebook after opening it with ``jupyter notebook`` command from ``/src/package`` directory.
+Example code can be run through Jupyter Notebook. Do this by entering the ``jupyter notebook`` command from the ``/src/package/`` directory. The tool is designed to be used through a GUI, not from the console.
 
 Demo
 ----
-AIM-CU can also be run through the demo available at https://huggingface.co/spaces/didsr/AIM-CU. If Space is paused, click on Restart button.
+AIM-CU can also be run through the demo available at https://huggingface.co/spaces/didsr/AIM-CU. If Space is paused, click on Restart button. Note: this Space uses a custom Docker container; build may break due to latest package updates pulled by HuggingFace.
+
+Usability
+---------
+* Example AI output CSV file is available as `config/spec-60-60.csv <config/spec-60-60.csv>`_ to be uploaded in monitoring phase.
+
+* Workflow instruction to run the tool is available at bottom-left of UI.
+
+* Sample UI output is available at `assets/ui.png <assets/ui.png>`_.
+
+* Setting ``control:save_figure`` to ``true`` from `config.toml <config/config.toml>`_ will save tables and plots in `figure/ <figure/>`_.
+
+* Running AIM-CU usually only takes a few seconds, and it does not require a GPU to run.
 
 Related References
 ------------------
@@ -58,7 +91,6 @@ Related References
 
 * Smriti Prathapan, Ravi K. Samala, Nathan Hadjiyski, Pierre‑François D’Haese, Nicholas Petrick, Jana Delfino, Fabien Maldonado, Brandon Nelson, Ghada Zamzmi, Phuong Nguyen, Yelena Yesha, and Berkman Sahiner, "Post-market Monitoring of AI-enabled Medical Devices for Radiology and Healthcare Applications" (FDA-UMiami Collaboration Poster, September 2023)
 
-
 Disclaimer
 ----------
 This software and documentation was developed at the Food and Drug Administration (FDA) by employees of the Federal Government in the course of their official duties. Pursuant to Title 17, Section 105 of the United States Code, this work is not subject to copyright protection and is in the public domain. Permission is hereby granted, free of charge, to any person obtaining a copy of the Software, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, or sell copies of the Software or derivatives, and to permit persons to whom the Software is furnished to do so. FDA assumes no responsibility whatsoever for use by other parties of the Software, its source code, documentation or compiled executables, and makes no guarantees, expressed or implied, about its quality, reliability, or any other characteristic. Further, use of this code in no way implies endorsement by the FDA or confers any advantage in regulatory decisions. Although this software can be redistributed and/or modified freely, we ask that any derivative works bear some notice that they are derived from it, and any modified versions bear some notice that they have been modified.

docs/build/html/_sources/index.rst.txt

@@ -8,40 +8,83 @@ AIM-CU documentation
 
 A CUSUM-based tool for AI Monitoring
 
-AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU). AIM-CU computes:
+Monitoring a clinically deployed AI device to detect performance drift is an essential step to ensure the safety and effectiveness of AI. 
+
+AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU).
+
+AIM-CU computes:
 
 * The parameter choices for change-point detection based on an acceptable false alarm rate
 * Detection delay estimates for a given displacement of the performance metric from the target for those parameter choices.
 
+System setup
+------------
+Make sure R is installed in the system. Instructions for linux (the below setup is only performed in linux):
+
+.. code-block:: shell
+
+    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc |  tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
+    add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
+    apt-get install -y --no-install-recommends r-base r-base-dev
+
+    # setup R configs
+    echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.us.r-project.org'; options(repos = r);" > ~/.Rprofile
+    Rscript -e "install.packages('ggplot2')"
+    Rscript -e "install.packages('hexbin')"
+    Rscript -e "install.packages('lazyeval')"
+    Rscript -e "install.packages('cusumcharter')"
+    Rscript -e "install.packages('RcppCNPy')"
+    Rscript -e "install.packages('spc')"
+
 Code execution
-------------------
+--------------
 Clone AIM-CU repository.
 
 .. code-block:: shell
 
-   git clone https://github.com/DIDSR/AIM-CU.git
+    git clone https://github.com/DIDSR/AIM-CU.git
 
 Run the following commands to install required dependencies (Python = 3.10 is used).
 
 .. code-block:: shell
 
-   apt-get -y install python3
-   apt-get -y install pip
-   cd AIM-CU
-   pip install -r requirements.txt
+    apt-get -y install python3
+    apt-get -y install pip
+    cd AIM-CU
+    pip install -r requirements.txt
 
 Run AIM-CU.
 
 .. code-block:: shell
 
-   python3 app.py
+    cd src/package
+    python3 app.py
 
 Open the URL http://0.0.0.0:7860 that is running the AIM-CU locally.
 
+Example code execution
+----------------------
+Example code can be run in a Jupyter Notebook after opening it with ``jupyter notebook`` command from ``/src/package/`` directory. The tool is designed to used through UI, not from console.
+
 Demo
-------------------
+----
 AIM-CU can also be run through the demo available at https://huggingface.co/spaces/didsr/AIM-CU. If Space is paused, click on Restart button.
 
+Usability
+---------
+* Example AI output CSV file is available as `config/spec-60-60.csv <config/spec-60-60.csv>`_ to be uploaded in monitoring phase.
+
+* Workflow instruction to run the tool is available at bottom-left of UI.
+
+* Sample UI output is available at `assets/ui.png <assets/ui.png>`_.
+
+* Setting ``control:save_figure`` to ``true`` from `config.toml <config/config.toml>`_ will save tables and plots in `figure/ <figure/>`_.
+
+* Running AIM-CU does not take time longer than a few seconds, and it does not require GPU.
+
+AIM-CU
+------
+
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
@@ -49,4 +92,8 @@ AIM-CU can also be run through the demo available at https://huggingface.co/spac
    ref_method
    ref_cusum
    ref_theoretical
-   ref_utils
+   ref_utils
+
+Disclaimer
+----------
+This software and documentation was developed at the Food and Drug Administration (FDA) by employees of the Federal Government in the course of their official duties. Pursuant to Title 17, Section 105 of the United States Code, this work is not subject to copyright protection and is in the public domain. Permission is hereby granted, free of charge, to any person obtaining a copy of the Software, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, or sell copies of the Software or derivatives, and to permit persons to whom the Software is furnished to do so. FDA assumes no responsibility whatsoever for use by other parties of the Software, its source code, documentation or compiled executables, and makes no guarantees, expressed or implied, about its quality, reliability, or any other characteristic. Further, use of this code in no way implies endorsement by the FDA or confers any advantage in regulatory decisions. Although this software can be redistributed and/or modified freely, we ask that any derivative works bear some notice that they are derived from it, and any modified versions bear some notice that they have been modified.

docs/build/html/index.html

@@ -36,11 +36,31 @@
   <section id="aim-cu-documentation">
 <h1>AIM-CU documentation<a class="headerlink" href="#aim-cu-documentation" title="Link to this heading">¶</a></h1>
 <p>A CUSUM-based tool for AI Monitoring</p>
-<p>AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU). AIM-CU computes:</p>
+<p>Monitoring a clinically deployed AI device to detect performance drift is an essential step to ensure the safety and effectiveness of AI.</p>
+<p>AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU).</p>
+<p>AIM-CU computes:</p>
 <ul class="simple">
 <li><p>The parameter choices for change-point detection based on an acceptable false alarm rate</p></li>
 <li><p>Detection delay estimates for a given displacement of the performance metric from the target for those parameter choices.</p></li>
 </ul>
+<section id="system-setup">
+<h2>System setup<a class="headerlink" href="#system-setup" title="Link to this heading">¶</a></h2>
+<p>Make sure R is installed in the system. Instructions for linux (the below setup is only performed in linux):</p>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>wget<span class="w"> </span>-qO-<span class="w"> </span>https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc<span class="w"> </span><span class="p">|</span><span class="w">  </span>tee<span class="w"> </span>-a<span class="w"> </span>/etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
+add-apt-repository<span class="w"> </span><span class="s2">&quot;deb https://cloud.r-project.org/bin/linux/ubuntu </span><span class="k">$(</span>lsb_release<span class="w"> </span>-cs<span class="k">)</span><span class="s2">-cran40/&quot;</span>
+apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>--no-install-recommends<span class="w"> </span>r-base<span class="w"> </span>r-base-dev
+
+<span class="c1"># setup R configs</span>
+<span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;r &lt;- getOption(&#39;repos&#39;); r[&#39;CRAN&#39;] &lt;- &#39;http://cran.us.r-project.org&#39;; options(repos = r);&quot;</span><span class="w"> </span>&gt;<span class="w"> </span>~/.Rprofile
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;ggplot2&#39;)&quot;</span>
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;hexbin&#39;)&quot;</span>
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;lazyeval&#39;)&quot;</span>
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;cusumcharter&#39;)&quot;</span>
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;RcppCNPy&#39;)&quot;</span>
+Rscript<span class="w"> </span>-e<span class="w"> </span><span class="s2">&quot;install.packages(&#39;spc&#39;)&quot;</span>
+</pre></div>
+</div>
+</section>
 <section id="code-execution">
 <h2>Code execution<a class="headerlink" href="#code-execution" title="Link to this heading">¶</a></h2>
 <p>Clone AIM-CU repository.</p>
@@ -55,14 +75,32 @@ pip<span class="w"> </span>install<span class="w"> </span>-r<span class="w"> </s
 </pre></div>
 </div>
 <p>Run AIM-CU.</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>python3<span class="w"> </span>app.py
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span><span class="w"> </span>src/package
+python3<span class="w"> </span>app.py
 </pre></div>
 </div>
 <p>Open the URL <a class="reference external" href="http://0.0.0.0:7860">http://0.0.0.0:7860</a> that is running the AIM-CU locally.</p>
 </section>
+<section id="example-code-execution">
+<h2>Example code execution<a class="headerlink" href="#example-code-execution" title="Link to this heading">¶</a></h2>
+<p>Example code can be run in a Jupyter Notebook after opening it with <code class="docutils literal notranslate"><span class="pre">jupyter</span> <span class="pre">notebook</span></code> command from <code class="docutils literal notranslate"><span class="pre">/src/package/</span></code> directory. The tool is designed to used through UI, not from console.</p>
+</section>
 <section id="demo">
 <h2>Demo<a class="headerlink" href="#demo" title="Link to this heading">¶</a></h2>
 <p>AIM-CU can also be run through the demo available at <a class="reference external" href="https://huggingface.co/spaces/didsr/AIM-CU">https://huggingface.co/spaces/didsr/AIM-CU</a>. If Space is paused, click on Restart button.</p>
+</section>
+<section id="usability">
+<h2>Usability<a class="headerlink" href="#usability" title="Link to this heading">¶</a></h2>
+<ul class="simple">
+<li><p>Example AI output CSV file is available as <a class="reference external" href="config/spec-60-60.csv">config/spec-60-60.csv</a> to be uploaded in monitoring phase.</p></li>
+<li><p>Workflow instruction to run the tool is available at bottom-left of UI.</p></li>
+<li><p>Sample UI output is available at <a class="reference external" href="assets/ui.png">assets/ui.png</a>.</p></li>
+<li><p>Setting <code class="docutils literal notranslate"><span class="pre">control:save_figure</span></code> to <code class="docutils literal notranslate"><span class="pre">true</span></code> from <a class="reference external" href="config/config.toml">config.toml</a> will save tables and plots in <a class="reference external" href="figure/">figure/</a>.</p></li>
+<li><p>Running AIM-CU does not take time longer than a few seconds, and it does not require GPU.</p></li>
+</ul>
+</section>
+<section id="aim-cu">
+<h2>AIM-CU<a class="headerlink" href="#aim-cu" title="Link to this heading">¶</a></h2>
 <div class="toctree-wrapper compound">
 <p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
 <ul>
@@ -91,6 +129,10 @@ pip<span class="w"> </span>install<span class="w"> </span>-r<span class="w"> </s
 </ul>
 </div>
 </section>
+<section id="disclaimer">
+<h2>Disclaimer<a class="headerlink" href="#disclaimer" title="Link to this heading">¶</a></h2>
+<p>This software and documentation was developed at the Food and Drug Administration (FDA) by employees of the Federal Government in the course of their official duties. Pursuant to Title 17, Section 105 of the United States Code, this work is not subject to copyright protection and is in the public domain. Permission is hereby granted, free of charge, to any person obtaining a copy of the Software, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, or sell copies of the Software or derivatives, and to permit persons to whom the Software is furnished to do so. FDA assumes no responsibility whatsoever for use by other parties of the Software, its source code, documentation or compiled executables, and makes no guarantees, expressed or implied, about its quality, reliability, or any other characteristic. Further, use of this code in no way implies endorsement by the FDA or confers any advantage in regulatory decisions. Although this software can be redistributed and/or modified freely, we ask that any derivative works bear some notice that they are derived from it, and any modified versions bear some notice that they have been modified.</p>
+</section>
 </section>
 
 

docs/build/html/searchindex.js

@@ -1 +1 @@
-Search.setIndex({"alltitles": {"AIM-CU documentation": [[0, null]], "ARLTheoretical": [[3, null]], "CUSUM": [[1, null]], "CUSUM chart": [[2, "cusum-chart"]], "CUSUM parameters": [[2, "cusum-parameters"], [2, "id1"]], "Code execution": [[0, "code-execution"]], "Contents:": [[0, null]], "Demo": [[0, "demo"]], "Methods": [[2, null]], "Utils": [[4, null]]}, "docnames": ["index", "ref_cusum", "ref_method", "ref_theoretical", "ref_utils"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst", "ref_cusum.rst", "ref_method.rst", "ref_theoretical.rst", "ref_utils.rst"], "indexentries": {"change_detection() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.change_detection", false]], "compute_cusum() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.compute_cusum", false]], "cusum (class in package.cusum)": [[1, "package.cusum.CUSUM", false]], "initialize() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.initialize", false]], "module": [[1, "module-package.cusum", false]], "package.cusum": [[1, "module-package.cusum", false]], "plot_cusum_plotly() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.plot_cusum_plotly", false]], "plot_input_metric_plotly() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.plot_input_metric_plotly", false]], "plot_input_metric_plotly_raw() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.plot_input_metric_plotly_raw", false]], "set_df_metric_csv() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.set_df_metric_csv", false]], "set_df_metric_default() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.set_df_metric_default", false]], "set_init_stats() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.set_init_stats", false]], "set_timeline() (package.cusum.cusum method)": [[1, "package.cusum.CUSUM.set_timeline", false]]}, "objects": {"package": [[3, 0, 0, "-", "ARLTheoretical"], [1, 0, 0, "-", "cusum"], [4, 0, 0, "-", "utils"]], "package.ARLTheoretical": [[3, 1, 1, "", "get_ARL_1"], [3, 1, 1, "", "get_ARL_1_h_mu1_k"], [3, 1, 1, "", "get_ref_value"], [3, 1, 1, "", "get_ref_value_k"]], "package.cusum": [[1, 2, 1, "", "CUSUM"]], "package.cusum.CUSUM": [[1, 3, 1, "", "change_detection"], [1, 3, 1, "", "compute_cusum"], [1, 3, 1, "", "initialize"], [1, 3, 1, "", "plot_cusum_plotly"], [1, 3, 1, "", "plot_input_metric_plotly"], [1, 3, 1, "", "plot_input_metric_plotly_raw"], [1, 3, 1, "", "set_df_metric_csv"], [1, 3, 1, "", "set_df_metric_default"], [1, 3, 1, "", "set_init_stats"], [1, 3, 1, "", "set_timeline"]], "package.utils": [[4, 1, 1, "", "get_greattable_as_html"], [4, 1, 1, "", "populate_summary_table_ARL0_k"], [4, 1, 1, "", "populate_summary_table_ARL1_k"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "method", "Python method"]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:method"}, "terms": {"0": [0, 1, 2], "1": 2, "10": 0, "3": 0, "30": 1, "4": [1, 2], "5": [1, 2], "7860": 0, "A": [0, 1, 2], "If": 0, "In": 1, "The": [0, 2], "To": 2, "_": 2, "accept": 0, "accumul": 2, "across": 1, "ai": [0, 1], "alarm": 0, "along": 3, "also": 0, "an": [0, 1], "app": 0, "apt": 0, "ar": [1, 2], "arl0": [3, 4], "arl1": [3, 4], "arl_0": [2, 3], "arl_1": [2, 3], "arltheoret": [0, 4], "assign": 1, "author": [1, 3], "avail": 0, "awai": 2, "ax": 1, "base": 0, "befor": 2, "being": 2, "between": 4, "button": 0, "calcul": [1, 2, 3], "can": 0, "cd": 0, "chang": [0, 1, 2], "change_detect": 1, "chart": 0, "choic": [0, 2], "class": 1, "click": 0, "clone": 0, "co": 0, "com": 0, "comma": 1, "command": 0, "comput": [0, 1, 2], "compute_cusum": 1, "configur": 1, "consid": 1, "control": [1, 2], "csv": 1, "cumul": [0, 1, 2], "cusum": 0, "d": 2, "dai": 1, "data": [1, 3, 4], "data_csv": 1, "datafram": [1, 3, 4], "default": [1, 2], "delai": [0, 3], "denot": 2, "depend": 0, "descript": 2, "detect": [0, 1, 2, 3], "determin": 2, "deviat": [1, 2], "df": 4, "dict_arl0_k": [3, 4], "dictionari": [3, 4], "didsr": 0, "differ": [2, 4], "dimension": 1, "displac": 0, "drift": 2, "e": 2, "estim": 0, "exampl": 1, "fals": [0, 2], "figur": 1, "file": 1, "float": [1, 3, 4], "follow": 0, "format": 4, "from": [0, 2, 4], "function": 1, "get": [0, 3, 4], "get_arl_1": [0, 3], "get_arl_1_h_mu1_k": [0, 3], "get_greattable_as_html": [0, 4], "get_ref_valu": [0, 3], "get_ref_value_k": [0, 3], "git": 0, "github": 0, "given": [0, 3], "go": 1, "graph": 1, "great_tabl": 4, "gt": 4, "h": [2, 3, 4], "handl": 4, "hat": 2, "hi": 2, "html": 4, "http": 0, "huggingfac": 0, "i": [0, 1, 2, 3], "individu": 2, "init_dai": 1, "initi": 1, "input": 1, "instal": 0, "int": 1, "intend": 3, "interest": [1, 2], "its": [1, 4], "k": [1, 2, 3, 4], "label": 1, "let": 2, "limit": 2, "list": [1, 3], "list_arl_0": 3, "lo": 2, "local": 0, "magnitud": [1, 2], "map": 4, "max": 2, "mean": [1, 2, 3], "method": 0, "metric": [0, 1, 2], "monitor": [0, 1, 2], "mu": 2, "mu1": 3, "mu_": 2, "mu_0": 1, "multiindex": 4, "ndarrai": 1, "neg": [1, 2], "none": 1, "normal": [1, 2, 3, 4], "normalized_ref_valu": 1, "normalized_threshold": 1, "np": 1, "number": [1, 2], "object": 1, "observ": [1, 2], "one": [1, 2], "open": 0, "oper": 4, "option": 1, "ordereddict": [3, 4], "output": 1, "packag": [1, 3, 4], "panda": 4, "paramet": [0, 1, 3, 4], "paus": 0, "pd": [3, 4], "perform": [0, 1, 2], "phase": 1, "pip": 0, "plot": 1, "plot_cusum_plotli": 1, "plot_input_metric_plotli": 1, "plot_input_metric_plotly_raw": 1, "plotli": 1, "point": 0, "popul": 4, "populate_summary_table_arl0_k": [0, 4], "populate_summary_table_arl1_k": [0, 4], "posit": [1, 2], "prathapan": [1, 3], "pre_change_dai": 1, "process": [1, 2], "provid": [1, 3], "py": 0, "python": 0, "python3": 0, "quantiti": 2, "r": 0, "rate": 0, "read": 1, "refer": [1, 2, 3], "relat": [1, 2], "rendera": 4, "repositori": 0, "requir": 0, "respect": 4, "restart": 0, "return": [1, 3, 4], "run": 0, "s_": 2, "s_hi": 2, "s_lo": 2, "scatter": 1, "scheme": 2, "separ": 1, "set": 1, "set_df_metric_csv": 1, "set_df_metric_default": 1, "set_init_stat": 1, "set_timelin": 1, "shift": 3, "shift_in_mean": 3, "side": 2, "sigma_": 2, "signal": 2, "smriti": [1, 3], "space": 0, "specif": 4, "stabl": 1, "standard": [1, 2], "statist": 0, "structur": 1, "sum": [0, 1, 2], "summary_table_df_arl0_k": 4, "summary_table_df_arl1_k": 4, "tabl": [3, 4], "target": [0, 2], "textfileread": 1, "thi": 2, "those": 0, "threshold": [1, 2, 3, 4], "through": 0, "time": 2, "timelin": 1, "tool": 0, "true": 2, "tupl": [1, 3], "two": [1, 2], "txt": 0, "type": [1, 3, 4], "unit": [1, 2], "url": 0, "us": [0, 1], "util": 0, "valu": [1, 2, 3, 4], "when": [1, 2], "where": [2, 3], "which": 2, "x": 1, "x_d": 2, "y": 0, "\u03bc_in": 2, "\u03c3_in": 2}, "titles": ["AIM-CU documentation", "CUSUM", "Methods", "ARLTheoretical", "Utils"], "titleterms": {"aim": 0, "arltheoret": 3, "chart": 2, "code": 0, "content": 0, "cu": 0, "cusum": [1, 2], "demo": 0, "document": 0, "execut": 0, "method": 2, "paramet": 2, "util": 4}})
+Search.setIndex({"alltitles": {"AIM-CU": [[0, "aim-cu"]], "AIM-CU documentation": [[0, null]], "ARLTheoretical": [[3, null]], "CUSUM": [[1, null]], "CUSUM chart": [[2, "cusum-chart"]], "CUSUM parameters": [[2, "cusum-parameters"], [2, "id1"]], "Code execution": [[0, "code-execution"]], "Contents:": [[0, null]], "Demo": [[0, "demo"]], "Disclaimer": [[0, "disclaimer"]], "Example code execution": [[0, "example-code-execution"]], "Methods": [[2, null]], "System setup": [[0, "system-setup"]], "Usability": [[0, "usability"]], "Utils": [[4, null]]}, "docnames": ["index", "ref_cusum", "ref_method", "ref_theoretical", "ref_utils"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst", "ref_cusum.rst", "ref_method.rst", "ref_theoretical.rst", "ref_utils.rst"], "indexentries": {}, "objects": {"package": [[3, 0, 0, "-", "ARLTheoretical"], [1, 0, 0, "-", "cusum"], [4, 0, 0, "-", "utils"]], "package.ARLTheoretical": [[3, 1, 1, "", "get_ARL_1"], [3, 1, 1, "", "get_ARL_1_h_mu1_k"], [3, 1, 1, "", "get_ref_value"], [3, 1, 1, "", "get_ref_value_k"]], "package.cusum": [[1, 2, 1, "", "CUSUM"]], "package.cusum.CUSUM": [[1, 3, 1, "", "change_detection"], [1, 3, 1, "", "compute_cusum"], [1, 3, 1, "", "initialize"], [1, 3, 1, "", "plot_cusum_plotly"], [1, 3, 1, "", "plot_input_metric_plotly"], [1, 3, 1, "", "plot_input_metric_plotly_raw"], [1, 3, 1, "", "set_df_metric_csv"], [1, 3, 1, "", "set_df_metric_default"], [1, 3, 1, "", "set_init_stats"], [1, 3, 1, "", "set_timeline"]], "package.utils": [[4, 1, 1, "", "get_greattable_as_html"], [4, 1, 1, "", "populate_summary_table_ARL0_k"], [4, 1, 1, "", "populate_summary_table_ARL1_k"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "method", "Python method"]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:method"}, "terms": {"0": [0, 1, 2], "1": 2, "10": 0, "105": 0, "17": 0, "3": 0, "30": 1, "4": [1, 2], "5": [1, 2], "60": 0, "7860": 0, "A": [0, 1, 2], "If": 0, "In": 1, "The": [0, 2], "To": 2, "_": 2, "about": 0, "accept": 0, "accumul": 2, "across": 1, "add": 0, "administr": 0, "advantag": 0, "after": 0, "ai": [0, 1], "alarm": 0, "along": 3, "also": 0, "although": 0, "an": [0, 1], "ani": 0, "app": 0, "apt": 0, "ar": [0, 1, 2], "arl0": [3, 4], "arl1": [3, 4], "arl_0": [2, 3], "arl_1": [2, 3], "arltheoret": [0, 4], "asc": 0, "ask": 0, "asset": 0, "assign": 1, "assum": 0, "author": [1, 3], "avail": 0, "awai": 2, "ax": 1, "base": 0, "bear": 0, "been": 0, "befor": 2, "being": 2, "below": 0, "between": 4, "bin": 0, "bottom": 0, "button": 0, "c": 0, "calcul": [1, 2, 3], "can": 0, "cd": 0, "chang": [0, 1, 2], "change_detect": 1, "characterist": 0, "charg": 0, "chart": 0, "choic": [0, 2], "class": 1, "click": 0, "clinic": 0, "clone": 0, "cloud": 0, "co": 0, "com": 0, "comma": 1, "command": 0, "compil": 0, "comput": [0, 1, 2], "compute_cusum": 1, "confer": 0, "config": 0, "configur": 1, "consid": 1, "consol": 0, "control": [0, 1, 2], "copi": 0, "copyright": 0, "cours": 0, "cran": 0, "cran40": 0, "cran_ubuntu_kei": 0, "csv": [0, 1], "cumul": [0, 1, 2], "cusum": 0, "cusumchart": 0, "d": [0, 2], "dai": 1, "data": [1, 3, 4], "data_csv": 1, "datafram": [1, 3, 4], "deal": 0, "deb": 0, "decis": 0, "default": [1, 2], "delai": [0, 3], "denot": 2, "depend": 0, "deploi": 0, "deriv": 0, "descript": 2, "design": 0, "detect": [0, 1, 2, 3], "determin": 2, "dev": 0, "develop": 0, "deviat": [1, 2], "devic": 0, "df": 4, "dict_arl0_k": [3, 4], "dictionari": [3, 4], "didsr": 0, "differ": [2, 4], "dimension": 1, "directori": 0, "displac": 0, "distribut": 0, "do": 0, "doe": 0, "domain": 0, "drift": [0, 2], "drug": 0, "duti": 0, "e": [0, 2], "echo": 0, "effect": 0, "employe": 0, "endors": 0, "ensur": 0, "essenti": 0, "estim": 0, "etc": 0, "exampl": 1, "express": 0, "fals": [0, 2], "fda": 0, "feder": 0, "few": 0, "figur": [0, 1], "file": [0, 1], "float": [1, 3, 4], "follow": 0, "food": 0, "format": 4, "free": 0, "freeli": 0, "from": [0, 2, 4], "function": 1, "furnish": 0, "further": 0, "get": [0, 3, 4], "get_arl_1": [0, 3], "get_arl_1_h_mu1_k": [0, 3], "get_greattable_as_html": [0, 4], "get_ref_valu": [0, 3], "get_ref_value_k": [0, 3], "getopt": 0, "ggplot2": 0, "git": 0, "github": 0, "given": [0, 3], "go": 1, "govern": 0, "gpg": 0, "gpu": 0, "grant": 0, "graph": 1, "great_tabl": 4, "gt": 4, "guarante": 0, "h": [2, 3, 4], "handl": 4, "hat": 2, "have": 0, "herebi": 0, "hexbin": 0, "hi": 2, "html": 4, "http": 0, "huggingfac": 0, "i": [0, 1, 2, 3], "impli": 0, "includ": 0, "individu": 2, "init_dai": 1, "initi": 1, "input": 1, "instal": 0, "instruct": 0, "int": 1, "intend": 3, "interest": [1, 2], "its": [0, 1, 4], "jupyt": 0, "k": [1, 2, 3, 4], "label": 1, "lazyev": 0, "left": 0, "let": 2, "limit": [0, 2], "linux": 0, "list": [1, 3], "list_arl_0": 3, "lo": 2, "local": 0, "longer": 0, "lsb_releas": 0, "magnitud": [1, 2], "make": 0, "map": 4, "marutter_pubkei": 0, "max": 2, "mean": [1, 2, 3], "merg": 0, "method": 0, "metric": [0, 1, 2], "modifi": 0, "monitor": [0, 1, 2], "mu": 2, "mu1": 3, "mu_": 2, "mu_0": 1, "multiindex": 4, "ndarrai": 1, "neg": [1, 2], "none": 1, "normal": [1, 2, 3, 4], "normalized_ref_valu": 1, "normalized_threshold": 1, "notebook": 0, "notic": 0, "np": 1, "number": [1, 2], "object": 1, "observ": [1, 2], "obtain": 0, "offici": 0, "one": [1, 2], "onli": 0, "open": 0, "oper": 4, "option": [0, 1], "ordereddict": [3, 4], "org": 0, "other": 0, "output": [0, 1], "packag": [0, 1, 3, 4], "panda": 4, "paramet": [0, 1, 3, 4], "parti": 0, "paus": 0, "pd": [3, 4], "perform": [0, 1, 2], "permiss": 0, "permit": 0, "person": 0, "phase": [0, 1], "pip": 0, "plot": [0, 1], "plot_cusum_plotli": 1, "plot_input_metric_plotli": 1, "plot_input_metric_plotly_raw": 1, "plotli": 1, "png": 0, "point": 0, "popul": 4, "populate_summary_table_arl0_k": [0, 4], "populate_summary_table_arl1_k": [0, 4], "posit": [1, 2], "prathapan": [1, 3], "pre_change_dai": 1, "process": [1, 2], "project": 0, "protect": 0, "provid": [1, 3], "public": 0, "publish": 0, "pursuant": 0, "py": 0, "python": 0, "python3": 0, "qo": 0, "qualiti": 0, "quantiti": 2, "r": 0, "rate": 0, "rcppcnpy": 0, "read": 1, "recommend": 0, "redistribut": 0, "refer": [1, 2, 3], "regulatori": 0, "relat": [1, 2], "reliabl": 0, "rendera": 4, "repo": 0, "repositori": 0, "requir": 0, "respect": 4, "respons": 0, "restart": 0, "restrict": 0, "return": [1, 3, 4], "right": 0, "rprofil": 0, "rscript": 0, "run": 0, "s_": 2, "s_hi": 2, "s_lo": 2, "safeti": 0, "sampl": 0, "save": 0, "save_figur": 0, "scatter": 1, "scheme": 2, "second": 0, "section": 0, "sell": 0, "separ": 1, "set": [0, 1], "set_df_metric_csv": 1, "set_df_metric_default": 1, "set_init_stat": 1, "set_timelin": 1, "shift": 3, "shift_in_mean": 3, "side": 2, "sigma_": 2, "signal": 2, "smriti": [1, 3], "so": 0, "softwar": 0, "some": 0, "sourc": 0, "space": 0, "spc": 0, "spec": 0, "specif": 4, "src": 0, "stabl": 1, "standard": [1, 2], "state": 0, "statist": 0, "step": 0, "structur": 1, "subject": 0, "sublicens": 0, "sum": [0, 1, 2], "summary_table_df_arl0_k": 4, "summary_table_df_arl1_k": 4, "sure": 0, "tabl": [0, 3, 4], "take": 0, "target": [0, 2], "tee": 0, "textfileread": 1, "than": 0, "thei": 0, "thi": [0, 2], "those": 0, "threshold": [1, 2, 3, 4], "through": 0, "time": [0, 2], "timelin": 1, "titl": 0, "toml": 0, "tool": 0, "true": [0, 2], "trust": 0, "tupl": [1, 3], "two": [1, 2], "txt": 0, "type": [1, 3, 4], "u": 0, "ubuntu": 0, "ui": 0, "unit": [0, 1, 2], "upload": 0, "url": 0, "us": [0, 1], "util": 0, "valu": [1, 2, 3, 4], "version": 0, "wa": 0, "wai": 0, "we": 0, "wget": 0, "whatsoev": 0, "when": [1, 2], "where": [2, 3], "which": 2, "whom": 0, "without": 0, "work": 0, "workflow": 0, "x": 1, "x_d": 2, "y": 0, "\u03bc_in": 2, "\u03c3_in": 2}, "titles": ["AIM-CU documentation", "CUSUM", "Methods", "ARLTheoretical", "Utils"], "titleterms": {"aim": 0, "arltheoret": 3, "chart": 2, "code": 0, "content": 0, "cu": 0, "cusum": [1, 2], "demo": 0, "disclaim": 0, "document": 0, "exampl": 0, "execut": 0, "method": 2, "paramet": 2, "setup": 0, "system": 0, "usabl": 0, "util": 4}})

docs/build/latex/aim-cu.pdf

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93dfcb8bab790019e595646e65c50e4d9dd98f6edac6c5460a9c8e1714f75ab9
-size 214164
+oid sha256:43d5c4317b6206ef1cbbecf90ef47516a795e6fccecd683b267962e9cb3c6c64
+size 242546

docs/source/index.rst

@@ -8,40 +8,83 @@ AIM-CU documentation
 
 A CUSUM-based tool for AI Monitoring
 
-AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU). AIM-CU computes:
+Monitoring a clinically deployed AI device to detect performance drift is an essential step to ensure the safety and effectiveness of AI. 
+
+AIM-CU is a statistical tool for AI monitoring using cumulative sum (AIM-CU).
+
+AIM-CU computes:
 
 * The parameter choices for change-point detection based on an acceptable false alarm rate
 * Detection delay estimates for a given displacement of the performance metric from the target for those parameter choices.
 
+System setup
+------------
+Make sure R is installed in the system. Instructions for linux (the below setup is only performed in linux):
+
+.. code-block:: shell
+
+    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc |  tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
+    add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
+    apt-get install -y --no-install-recommends r-base r-base-dev
+
+    # setup R configs
+    echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.us.r-project.org'; options(repos = r);" > ~/.Rprofile
+    Rscript -e "install.packages('ggplot2')"
+    Rscript -e "install.packages('hexbin')"
+    Rscript -e "install.packages('lazyeval')"
+    Rscript -e "install.packages('cusumcharter')"
+    Rscript -e "install.packages('RcppCNPy')"
+    Rscript -e "install.packages('spc')"
+
 Code execution
-------------------
+--------------
 Clone AIM-CU repository.
 
 .. code-block:: shell
 
-   git clone https://github.com/DIDSR/AIM-CU.git
+    git clone https://github.com/DIDSR/AIM-CU.git
 
 Run the following commands to install required dependencies (Python = 3.10 is used).
 
 .. code-block:: shell
 
-   apt-get -y install python3
-   apt-get -y install pip
-   cd AIM-CU
-   pip install -r requirements.txt
+    apt-get -y install python3
+    apt-get -y install pip
+    cd AIM-CU
+    pip install -r requirements.txt
 
 Run AIM-CU.
 
 .. code-block:: shell
 
-   python3 app.py
+    cd src/package
+    python3 app.py
 
 Open the URL http://0.0.0.0:7860 that is running the AIM-CU locally.
 
+Example code execution
+----------------------
+Example code can be run in a Jupyter Notebook after opening it with ``jupyter notebook`` command from ``/src/package/`` directory. The tool is designed to used through UI, not from console.
+
 Demo
-------------------
+----
 AIM-CU can also be run through the demo available at https://huggingface.co/spaces/didsr/AIM-CU. If Space is paused, click on Restart button.
 
+Usability
+---------
+* Example AI output CSV file is available as `config/spec-60-60.csv <config/spec-60-60.csv>`_ to be uploaded in monitoring phase.
+
+* Workflow instruction to run the tool is available at bottom-left of UI.
+
+* Sample UI output is available at `assets/ui.png <assets/ui.png>`_.
+
+* Setting ``control:save_figure`` to ``true`` from `config.toml <config/config.toml>`_ will save tables and plots in `figure/ <figure/>`_.
+
+* Running AIM-CU does not take time longer than a few seconds, and it does not require GPU.
+
+AIM-CU
+------
+
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
@@ -49,4 +92,8 @@ AIM-CU can also be run through the demo available at https://huggingface.co/spac
    ref_method
    ref_cusum
    ref_theoretical
-   ref_utils
+   ref_utils
+
+Disclaimer
+----------
+This software and documentation was developed at the Food and Drug Administration (FDA) by employees of the Federal Government in the course of their official duties. Pursuant to Title 17, Section 105 of the United States Code, this work is not subject to copyright protection and is in the public domain. Permission is hereby granted, free of charge, to any person obtaining a copy of the Software, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, or sell copies of the Software or derivatives, and to permit persons to whom the Software is furnished to do so. FDA assumes no responsibility whatsoever for use by other parties of the Software, its source code, documentation or compiled executables, and makes no guarantees, expressed or implied, about its quality, reliability, or any other characteristic. Further, use of this code in no way implies endorsement by the FDA or confers any advantage in regulatory decisions. Although this software can be redistributed and/or modified freely, we ask that any derivative works bear some notice that they are derived from it, and any modified versions bear some notice that they have been modified.

src/package/ARLTheoretical.py

@@ -129,7 +129,7 @@ def get_ARL_1(
 utils = rpackages.importr("utils")
 spc = rpackages.importr("spc")
 # select a mirror for R packages
-utils.chooseCRANmirror(ind=1)  # select the first mirror in the list
+# utils.chooseCRANmirror(ind=1)  # select the first mirror in the list
 
 # R package names
 packnames = ("ggplot2", "hexbin", "lazyeval", "cusumcharter", "RcppCNPy", "spc")

src/package/app.py

@@ -1,5 +1,8 @@
 """
 Gradio user interface for AIM-CU
+
+Disclaimer:
+This software and documentation was developed at the Food and Drug Administration (FDA) by employees of the Federal Government in the course of their official duties. Pursuant to Title 17, Section 105 of the United States Code, this work is not subject to copyright protection and is in the public domain. Permission is hereby granted, free of charge, to any person obtaining a copy of the Software, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, or sell copies of the Software or derivatives, and to permit persons to whom the Software is furnished to do so. FDA assumes no responsibility whatsoever for use by other parties of the Software, its source code, documentation or compiled executables, and makes no guarantees, expressed or implied, about its quality, reliability, or any other characteristic. Further, use of this code in no way implies endorsement by the FDA or confers any advantage in regulatory decisions. Although this software can be redistributed and/or modified freely, we ask that any derivative works bear some notice that they are derived from it, and any modified versions bear some notice that they have been modified.
 """
 
 import os
@@ -21,11 +24,11 @@ def set_init_days(
     file_csv_metric: gr.File, init_days: str
 ) -> tuple[float, float, go.Figure]:
     """
-    Set initial days and get in-control mean and standard deviation.
+    Set number of baseline observations and get in-control mean and standard deviation.
 
     Args:
         file_csv_metric (gr.File): CSV file with metric data
-        init_days (str): initial days to calculate in-control mean and standard deviation
+        init_days (str): Number of baseline observations to calculate in-control mean and standard deviation
 
     Returns:
         tuple[float, float, go.Figure]: In-control mean and standard deviation, and observation data plot.
@@ -174,7 +177,7 @@ with gr.Blocks(
     with gr.Row():
         with gr.Column():
             gr.Markdown(f"""
-                        ### Phase I:
+                        ### Initialization:
                         """)  # noqa: F541
 
             gr.Markdown(f"""
@@ -183,12 +186,12 @@ with gr.Blocks(
 
             # load the CSV file with specifities across days
             csv_file_metric = gr.File(
-                label="Upload the AI output",
+                label="Upload the AI output (CSV file)",
             )
 
             with gr.Row():
                 with gr.Column():
-                    init_days = gr.Textbox(label="Initial days", placeholder="30")
+                    init_days = gr.Textbox(label="Number of baseline observations", placeholder="30")
                 with gr.Column():
                     button_calculate_incontrol_params = gr.Button(
                         "Calculate parameters"
@@ -271,8 +274,8 @@ with gr.Blocks(
             gr.Markdown(f"""
                 ### Workflow:
                 Phase I:
-                - Upload the AI output.
-                - Enter initial days.
+                - Upload the AI output (CSV file).
+                - Enter number of baseline observations.
                 - Calculate parameters.
                 - Check parameter choices in Phase 1, for Phase 2. (optional)
                 
@@ -286,7 +289,7 @@ with gr.Blocks(
             )
         with gr.Column():
             gr.Markdown(f"""
-                        ### Phase II:
+                        ### Monitoring:
                         Performance drift detection plots, pre- and post-change distribution with respect to the performance drift detected.
                         """)  # noqa: F541
 

src/package/cusum.py

@@ -60,10 +60,10 @@ class CUSUM:
 
     def set_init_stats(self, init_days: int) -> None:
         """
-        Use initial days to calculate in-control mean and standard deviation.
+        Use number of baseline observations to calculate in-control mean and standard deviation.
 
         Args:
-            init_days (int, optional): Initial days when observations are considered stable. Defaults to 30.
+            init_days (int, optional): Number of baseline observations when observations are considered stable. Defaults to 30.
         """
         self.init_days = init_days
 
@@ -171,7 +171,7 @@ class CUSUM:
             normalized_ref_value (float, optional): Normalized reference value for detecting a unit standard deviation change in mean of the process. Defaults to 0.5.
             normalized_threshold (float, optional): Normalized threshold. Defaults to 4.
         """
-        self.pre_change_days = self.init_days  # This is the #initial days that we assume to be in-control - user enters or default = 30
+        self.pre_change_days = self.init_days  # This is the number of baseline observations that we assume to be in-control - user enters or default = 30
 
         ref_val = normalized_ref_value
         control_limit = normalized_threshold
@@ -313,7 +313,7 @@ class CUSUM:
         fig.add_vrect(
             x0=0,
             x1=self.init_days,
-            annotation_text="Initial days",
+            annotation_text="Baseline observations",
             annotation_position="top right",
             fillcolor="palegreen",
             opacity=0.25,
@@ -326,7 +326,7 @@ class CUSUM:
                 "font": {"size": font_size_title, "weight": "bold"},
             },
             xaxis_title={
-                "text": "Length of observations",
+                "text": "Time",
                 "font": {"size": font_size_legend, "weight": "bold"},
             },
             yaxis_title={
@@ -361,13 +361,7 @@ class CUSUM:
         y2 = self.data[self.pre_change_days : self.total_days]
         mean_y2 = np.mean(y2)
 
-        fig = make_subplots(
-            rows=1,
-            cols=2,
-            column_widths=[0.7, 0.3],
-            shared_yaxes=True,
-            horizontal_spacing=0.02,
-        )
+        fig = go.Figure()
 
         font_size_title = 20
         font_size_legend = 18
@@ -382,8 +376,6 @@ class CUSUM:
                 marker=dict(color="darkturquoise", size=10),
                 opacity=0.4,
             ),
-            row=1,
-            col=1,
         )
         fig.add_trace(
             go.Scatter(
@@ -394,8 +386,6 @@ class CUSUM:
                 marker=dict(color="coral", size=10),
                 opacity=0.4,
             ),
-            row=1,
-            col=1,
         )
 
         # add horizontal lines
@@ -407,8 +397,6 @@ class CUSUM:
                 name="In-control mean",
                 line=dict(color="darkturquoise", dash="dash"),
             ),
-            row=1,
-            col=1,
         )
         fig.add_trace(
             go.Scatter(
@@ -418,8 +406,6 @@ class CUSUM:
                 name="Out-of-control mean",
                 line=dict(color="coral", dash="dash"),
             ),
-            row=1,
-            col=1,
         )
 
         # add vertical line
@@ -432,17 +418,15 @@ class CUSUM:
                 line=dict(color="grey", dash="dash"),
                 # textfont=dict(size=18)
             ),
-            row=1,
-            col=1,
         )
 
         fig.update_layout(
             title={
-                "text": "Pre- and post-change observations and their respective histograms",
+                "text": "Pre- and post-change observations",
                 "font": {"size": font_size_title, "weight": "bold"},
             },
             xaxis_title={
-                "text": "Length of observations",
+                "text": "Time",
                 "font": {"size": font_size_legend, "weight": "bold"},
             },
             yaxis_title={
@@ -458,64 +442,64 @@ class CUSUM:
             legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1)
         )
 
-        # add histogram (like marginal histogram)
-        nbinsx = 15  # 6
-
-        # add subplots
-        fig.add_trace(
-            go.Histogram(
-                y=self.data[: self.pre_change_days],
-                nbinsy=nbinsx,
-                # name=f"""Pre-change S<sub>p</sub>""",
-                showlegend=False,
-                marker=dict(color="mediumturquoise"),
-                opacity=0.4,
-                orientation="h",
-            ),
-            row=1,
-            col=2,
-        )
-
-        fig.add_trace(
-            go.Histogram(
-                y=self.data[self.pre_change_days : self.total_days],
-                nbinsy=nbinsx,
-                # name=f"""Post-change S<sub>p</sub>""",
-                showlegend=False,
-                marker=dict(color="coral"),
-                opacity=0.4,
-                orientation="h",
-            ),
-            row=1,
-            col=2,
-        )
-
-        fig.add_trace(
-            go.Scatter(
-                x=[0, 20],  # [! y_max can should be used]
-                y=[
-                    np.mean(self.data[: self.pre_change_days]),
-                    np.mean(self.data[: self.pre_change_days]),
-                ],
-                mode="lines",
-                # name="Reference mean",
-                showlegend=False,
-                line=dict(color="mediumturquoise", dash="dash"),
-            ),
-            row=1,
-            col=2,
-        )
-
-        fig.update_xaxes(
-            title_text="Count",
-            title_font_size=font_size_legend,
-            title_font_weight="bold",
-            row=1,
-            col=2,
-        )
-
-        # update layout
-        fig.update_layout(barmode="overlay")
+        # # add histogram (like marginal histogram)
+        # nbinsx = 15  # 6
+
+        # # add subplots
+        # fig.add_trace(
+        #     go.Histogram(
+        #         y=self.data[: self.pre_change_days],
+        #         nbinsy=nbinsx,
+        #         # name=f"""Pre-change S<sub>p</sub>""",
+        #         showlegend=False,
+        #         marker=dict(color="mediumturquoise"),
+        #         opacity=0.4,
+        #         orientation="h",
+        #     ),
+        #     row=1,
+        #     col=2,
+        # )
+
+        # fig.add_trace(
+        #     go.Histogram(
+        #         y=self.data[self.pre_change_days : self.total_days],
+        #         nbinsy=nbinsx,
+        #         # name=f"""Post-change S<sub>p</sub>""",
+        #         showlegend=False,
+        #         marker=dict(color="coral"),
+        #         opacity=0.4,
+        #         orientation="h",
+        #     ),
+        #     row=1,
+        #     col=2,
+        # )
+
+        # fig.add_trace(
+        #     go.Scatter(
+        #         x=[0, 20],  # [! y_max can should be used]
+        #         y=[
+        #             np.mean(self.data[: self.pre_change_days]),
+        #             np.mean(self.data[: self.pre_change_days]),
+        #         ],
+        #         mode="lines",
+        #         # name="Reference mean",
+        #         showlegend=False,
+        #         line=dict(color="mediumturquoise", dash="dash"),
+        #     ),
+        #     row=1,
+        #     col=2,
+        # )
+
+        # fig.update_xaxes(
+        #     title_text="Count",
+        #     title_font_size=font_size_legend,
+        #     title_font_weight="bold",
+        #     row=1,
+        #     col=2,
+        # )
+
+        # # update layout
+        # fig.update_layout(barmode="overlay")
 
         if self.config["control"]["save_figure"] == "true":
             fig.write_image(
@@ -525,9 +509,9 @@ class CUSUM:
                             "../../", self.config["path_output"]["path_figure"]
                         )
                     ),
-                    "fig_plot_1.png",
+                    "fig_plot_data_distribution.png",
                 ),
-                scale=3,
+                scale=6,
             )
             print(
                 "Created",
@@ -537,7 +521,7 @@ class CUSUM:
                             "../../", self.config["path_output"]["path_figure"]
                         )
                     ),
-                    "fig_plot_1.png",
+                    "fig_plot_data_distribution.png",
                 ),
             )
 
@@ -555,14 +539,13 @@ class CUSUM:
         font_size_title = 20
         font_size_legend = 18
 
-        # add subplots
         fig.add_trace(
             go.Scatter(
                 x=list(range(len(self.S_hi))),
                 y=self.S_hi / self.in_std,
                 mode="lines",
                 name=f"""Positive changes (S<sub>hi</sub>)""",
-                marker=dict(color="greenyellow", size=10),
+                marker=dict(color="rgb(0, 209, 209)", size=10),
             )
         )
         fig.add_trace(
@@ -582,21 +565,7 @@ class CUSUM:
                 y=[self.H / self.in_std, self.H / self.in_std],
                 mode="lines",
                 name="Threshold (h)",
-                line=dict(color="skyblue", dash="dash"),
-            )
-        )
-
-        # add vertical line
-        fig.add_trace(
-            go.Scatter(
-                x=[self.pre_change_days, self.pre_change_days],
-                y=[
-                    0,
-                    np.max(self.S_lo / self.in_std),
-                ],
-                mode="lines",
-                name="Change-point",
-                line=dict(color="grey", dash="dash"),
+                line=dict(color="rgb(250, 0, 125)", dash="dash"),
             )
         )
 
@@ -606,7 +575,7 @@ class CUSUM:
                 "font": {"size": font_size_title, "weight": "bold"},
             },
             xaxis_title={
-                "text": "Length of observations",
+                "text": "Time",
                 "font": {"size": font_size_legend, "weight": "bold"},
             },
             yaxis_title={
@@ -616,7 +585,28 @@ class CUSUM:
             xaxis=dict(dtick=20),
         )
 
-        fig.update_layout(plot_bgcolor=self.config["color"]["blue_005"])
+
+        fig.add_shape(
+            type="rect",
+            x0=0, x1=60,
+            y0=0, y1=1,  # use relative values (0 to 1) for full y-range
+            xref="x", yref="paper",  # "paper" for full plot height
+            fillcolor=self.config["color"]["blue_005"],
+            opacity=0.8,
+            layer="below",
+            line_width=0,
+        )
+
+        fig.add_shape(
+            type="rect",
+            x0=60, x1=len(self.S_lo),  # x1=1 means extend to right edge of plot (paper coordinates)
+            y0=0, y1=1,
+            xref="x", yref="paper",
+            fillcolor="rgb(253, 243, 235)",
+            opacity=0.8,
+            layer="below",
+            line_width=0,
+        )
 
         fig.update_layout(
             legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1)
@@ -630,9 +620,9 @@ class CUSUM:
                             "../../", self.config["path_output"]["path_figure"]
                         )
                     ),
-                    "fig_plot_2.png",
+                    "fig_plot_cusum_chart.png",
                 ),
-                scale=3,
+                scale=6,
             )
             print(
                 "Created",
@@ -642,7 +632,7 @@ class CUSUM:
                             "../../", self.config["path_output"]["path_figure"]
                         )
                     ),
-                    "fig_plot_2.png",
+                    "fig_plot_cusum_chart.png",
                 ),
             )
 

src/package/utils.py

@@ -59,7 +59,7 @@ def populate_summary_table_ARL0_k(summary_table_df_ARL0_k: pd.DataFrame, h) -> g
         table_great_table_ARL0_k.save(
             os.path.abspath(
                 os.path.join(
-                    "../../", config["path_output"]["path_figure"], "table_ARL0_k.png"
+                    "../../", config["path_output"]["path_figure"], "fig_table_h_arl0_k.png"
                 )
             ),
             scale=3,
@@ -69,7 +69,7 @@ def populate_summary_table_ARL0_k(summary_table_df_ARL0_k: pd.DataFrame, h) -> g
             "Created",
             os.path.abspath(
                 os.path.join(
-                    "../../", config["path_output"]["path_figure"], "table_ARL0_k.png"
+                    "../../", config["path_output"]["path_figure"], "fig_table_h_arl0_k.png"
                 )
             ),
         )
@@ -127,7 +127,7 @@ def populate_summary_table_ARL1_k(
         table_great_table_ARL1_k.save(
             os.path.abspath(
                 os.path.join(
-                    "../../", config["path_output"]["path_figure"], "table_ARL1_k.png"
+                    "../../", config["path_output"]["path_figure"], "fig_table_h_k_arl1.png"
                 )
             ),
             scale=3,
@@ -137,7 +137,7 @@ def populate_summary_table_ARL1_k(
             "Created",
             os.path.abspath(
                 os.path.join(
-                    "../../", config["path_output"]["path_figure"], "table_ARL1_k.png"
+                    "../../", config["path_output"]["path_figure"], "fig_table_h_k_arl1.png"
                 )
             ),
         )

tests/test.py

@@ -23,6 +23,7 @@ class TestCUSUM(unittest.TestCase):
 
     def test_cusum(self):
         obj_cusum = CUSUM()
+        # obj_cusum.initialize()
 
         path_file_config = os.path.abspath("../config/config.toml")
         with open(os.path.abspath(path_file_config), "rb") as file_config:
@@ -30,26 +31,28 @@ class TestCUSUM(unittest.TestCase):
 
         ref_value = 0.5
         normalized_threshold = 4
-        pre_change_days = 60
 
         obj_cusum.df_metric = pd.read_csv(os.path.abspath("../config/spec-60-60.csv"))
         obj_cusum.data = obj_cusum.df_metric[obj_cusum.df_metric.columns[1]].to_numpy()
         obj_cusum.set_timeline(obj_cusum.data)
 
+        # Set initial days and get in-control mean and standard deviation
+        obj_cusum.set_init_stats(init_days=30)
+
+        # Detects a change in the process
         obj_cusum.change_detection(
-            pre_change_days=pre_change_days,
             normalized_ref_value=ref_value,
             normalized_threshold=normalized_threshold,
         )
 
         self.assertEqual(
-            obj_cusum.S_lo[-1], 2.31, "Cumulative (negative) sum does not match."
+            obj_cusum.S_lo[-1], 2.6, "Cumulative (negative) sum does not match."
         )
 
     def test_rpy2(self):
         arl_1 = get_ARL_1_h_mu1_k(h=4, k=0.2996, mu1=1.2)
 
-        self.assertEqual(arl_1, 4.4304, "Package rpy2 is not working properly")
+        self.assertEqual(arl_1, 4.43, "Package rpy2 is not working properly")
 
 
 if __name__ == "__main__":