update check_overall_compliance, remove comments

compliance_analysis.py

@@ -1,5 +1,5 @@
 import yaml
-from utils import set_type, set_operator_role_and_location, set_eu_market_status, check_within_scope
+from utils import set_type, set_operator_role_and_location, set_eu_market_status, check_within_scope_cc, check_within_scope_act
 
 # Create some variables we will use throughout our analysis
 
@@ -20,38 +20,30 @@ dispositive_variables = {
         "put_into_service": False
     },
     "intended_purposes": [],
+    "project_cc_pass": False,
+    "data_cc_pass": False,
+    "model_cc_pass": False,
+    "msg": []
 }
 
-# Here is the potential orchestrator function that I think is the key missing part:
-# 
-# def orchestrator():
-#
-#   -make sure there is at least one Project CC, one Data CC, and one Model CC -- need at least one of each 
-#   -do some administrative stuff to make your life easier like maybe getting all the files in the folder into a list, etc.
-#
-#   -Call set_dispositive_variables, passing in all the cards as the argument: 
-#       -This must loop through all the cards to set the dispositive_variables where applicable. There is no function for this yet. I can write it.
-#       -It must set the intended purposes by parsing them from the Project CC and. I wrote a utility function for this.
-#   -Optionally call the functions that check whethe the project is in scope of CC and in scope of the Act. These could also be called from run_compliance_analysis_on_project
-#   -Optionally check for prohibited practices. This has been commented out, but the functionality is there as-is. This could also be called from run_compliance_analysis_on_project
-#   
-#   Call run_compliance_analysis_on_project, passing in the sole Project CC as the argument
-#       -This must run the internal check of the project CC based on the dispositive_variables it has set. It is only partially doing this as-is. To finish the job, we must:
-#        -Be sure to run the check for all types of models and systems including AI systems without high risk, GPAI without systemic risk, GPAI with systemic risk. It is only doing high-risk AI systems at the moment.
-#
-#   Call run_compliance_analysis_on_model() *for all model CCs in the folder*, passing in the ai_project_type variable and maybe project_intended_purpose 
-#       -This should include a "cross comparison" of the intended uses listed in the model CC and the project_intended_purpose parsed from the Project CC, something that is not yet integrated 
-#       -This function must check if GPAI requirements are met, if that value for ai_project_type is passed in -- it does not yet do this   
-#
-#   Call run_compliance_analysis_on_data() *for all data CCs in the folder*, passing in the ai_project_type variable and maybe project_intended_purpose
-#       -This should include a "cross comparison" of the intended uses listed in the data CC and the project_intended_purpose parsed from the Project CC, something that is not yet integrated  
-#       -This function must check if GPAI requirements are met, if that value for ai_project_type is passed in -- it does not yet do this 
-#
-#   This function could also more gracefully handle the internal exits/reports and generate a single, digestible compliance report that
-#   tells the user where the compliance analysis failed. If we wanted to get really fancy, we could include error messages for each individual
-#   entry in the yaml files, possibly citing the part of the Act that they need to reference (currently in comments that user does not see)
-
-def run_compliance_analysis_on_project(project_cc_yaml): 
+# TODO tells the user where the compliance analysis failed
+# TODO cite article from yaml file as explanation
+
+def check_overall_compliance(dispositive_variables, cc_files):
+
+    # check intended purposes 
+    dispositive_variables = check_intended_purpose(dispositive_variables, cc_files)
+   
+    # for each model_cc and data_cc - run analysis with ref to project_cc
+    
+    dispositive_variables = run_compliance_analysis_on_data(dispositive_variables, data_cc_yaml)
+    dispositive_variables = run_compliance_analysis_on_model(dispositive_variables, model_cc_yaml)
+    
+    dispositive_variables = run_compliance_analysis_on_project(dispositive_variables, project_cc_yaml)
+
+    return dispositive_variables
+
+def run_compliance_analysis_on_project(dispositive_variables, project_cc_yaml): 
 
     # Determine project type (AI system vs. GPAI model) as well as operator type. We will use these for different things.
     project_type = set_type(dispositive_variables, project_cc_yaml)
@@ -70,8 +62,9 @@ def run_compliance_analysis_on_project(project_cc_yaml):
     else: 
         msg = ("Project is not within the scope of what is regulated by the Act.")
 
-    # TO-DO: reactivate the prohibited practices check below 
+    # TODO: reactivate the prohibited practices check below 
 
+    # TODO: fix and uncomment
     # # Check for prohibited practices. If any exist, the analysis is over.
     # if check_prohibited(project_cc_yaml) == True: 
     #     print("Project contains prohibited practices and is therefore non-compliant.")
@@ -107,13 +100,47 @@ def run_compliance_analysis_on_project(project_cc_yaml):
             if not value:
                 msg = ("Because of project-level characteristics, this high-risk AI system fails the accuracy, robustness, and cybersecurity requirements under Article 17.") 
 
-    # TO-DO: No matter where we land with an orchestrator function, this function must also check to the value it has set for both
+
+    # TODO
+    # # If the project is a GPAI model, check that is has met all the requirements for such systems: 
+
+    if gpai_model:
+
+    # # If the project is a GPAI model with systematic risk, check that is has additionally met all the requirements for such systems: 
+
+    # if gpai_model_systematic_risk:
+
+    # # Do this by examining the Project CC
+
+    #     for key, value in project_cc_yaml['gpai_obligations_for_systemic_risk_models']:
+    #         if not value:
+    #             msg = ("GPAI model with systematic risk fails the transparency requirements under Article 55.")
+
+    # Do this by examining the Project CC
+
+        for key, value in project_cc_yaml['gpai_model_obligations']:
+            if not value:
+                msg = ("GPAI model fails the transparency requirements under Article 53.")    
+
+
+    if gpai_model_systematic_risk:
+        for key, value in project_cc_yaml['gpai_models_with_systemic_risk_obligations']:
+
+
+    # if ai_system:
+    #     for key, value in project_cc_yaml['']:
+        # TODO to be included in project_cc
+        
+
+    # TODO: No matter where we land with an orchestrator function, this function must also check to the value it has set for both
     # GPAI models with and without systemic risk and then check to see if the relevant requirement have met if either of these values applies.
     # This will look a lot like what is happening above for high-risk AI systems. 
     
-    return msg
+    return dispositive_variables
 
-def run_compliance_analysis_on_data(data_cc_yaml, project_intended_purpose): # TO-DO: we probably have to pass ai_project_type and project_intended_purpose into this function
+def run_compliance_analysis_on_data(dispositive_variables, data_cc_yaml): 
+    
+    # TODO: we probably have to pass ai_project_type and project_intended_purpose into this function
     
     for key, value in data_cc_yaml['data_and_data_governance']:
         if not value:
@@ -128,16 +155,23 @@ def run_compliance_analysis_on_data(data_cc_yaml, project_intended_purpose): # T
         if not value:
             msg = (f"Because of the dataset represented by , this high-risk  AI system fails the quality management requirements under Article 17.")
 
-    # TO-DO: No matter where we land with an orchestrator function, this function must also check to the value that has been set for both
+    #             for key, value in data_cc_yaml['gpai_requirements']['gpai_requirements']:
+    #                 if not value:
+    #                     msg = (f"Because of the dataset represented by {filename}, this GPAI fails the transparency requirements under Article 53.")
+
+
+    # TODO: No matter where we land with an orchestrator function, this function must also check to the value that has been set for both
     # GPAI models with and without systemic risk and then check to see if the relevant requirements have met if either of these values applies.
     # Right now it is only checking high-risk AI system requirements. Another thing that we likely have to add here is the cross-comparison of the 
     # intended purposes. That might look like this:
     # if data_cc_yaml['intended_purpose'] not in intended_purposes:
     #   return false 
 
-    return msg
+    return dispositive_variables
+    
+def run_compliance_analysis_on_model(dispositive_variables, model_cc_yaml):  
     
-def run_compliance_analysis_on_model(model_cc_yaml, project_intended_purpose):  # TO-DO: we probably have to pass ai_project_type and project_intended_purpose into this function
+    # TODO: we probably have to pass ai_project_type and project_intended_purpose into this function
     
     for key, value in model_cc_yaml['risk_management_system']:
         if not value:
@@ -154,17 +188,26 @@ def run_compliance_analysis_on_model(model_cc_yaml, project_intended_purpose):
     for key, value in data_cc_yaml['quality_management_system']:
         if not value:
             msg = (f"Because of the model represented by , this high-risk  AI system fails the quality management requirements under Article 17.")
+
+
+    #             for key, value in model_cc_yaml['obligations_for_providers_of_gpai_models']:
+    #                 if not value:
+    #                     msg = (f"Because of the model represented by {filename}, this GPAI fails the transparency requirements under Article 53.")
+
+    #             for key, value in model_cc_yaml['obligations_for_providers_of_gpai_models_with_systemic_risk']:
+    #                 if not value:
+    #                     msg = (f"Because of the model represented by {filename}, this GPAI model with systematic risk fails the transparency requirements under Article 55.")
    
-    # TO-DO: No matter where we land with an orchestrator function, this function must also check to the value that has been set for both
+    # TODO: No matter where we land with an orchestrator function, this function must also check to the value that has been set for both
     # GPAI models with and without systemic risk and then check to see if the relevant requirements have met if either of these values applies.
     # Right now it is only checking high-risk AI system requirements. Another thing that we likely have to add here is the cross-comparison of the 
     # intended purposes. That might look like this:
     # if model_cc_yaml['intended_purpose'] not in intended_purposes:
     #   return false 
     
-    return msg
+    return dispositive_variables
 
-def check_intended_purpose():
+def check_intended_purpose(dispositive_variables, cc_files):
     
     # We want to run this function for everything classified as a high_risk_ai_system
     # We also need to run it for all 
@@ -214,74 +257,6 @@ def check_intended_purpose():
 
     # TODO return list of intended purpose 
 
-    return msg
-
-
-
-    # # If the project is a GPAI model, check that is has met all the requirements for such systems: 
-
-    # if gpai_model:
-
-    # # Do this by examining the Project CC
-
-    #     for key, value in project_cc_yaml['gpai_model_provider_obligations']:
-    #         if not value:
-    #             msg = ("GPAI model fails the transparency requirements under Article 53.")
-
-    # # Do this by examining any and all Data CCs too
-
-    #     for filename in os.listdir(folder_path):
-    #         # Check if the search word is in the filename
-    #         if "data_cc.md" in filename.lower():
-
-    #             # If it is, load the yaml
-
-    #             with open(folder_path + filename, 'r') as file:
-    #                 data_cc_yaml = yaml.safe_load(file)
-
-    #             for key, value in data_cc_yaml['gpai_requirements']['gpai_requirements']:
-    #                 if not value:
-    #                     msg = (f"Because of the dataset represented by {filename}, this GPAI fails the transparency requirements under Article 53.")
-
-    # # Do this by examining any and all Model CCs too
-    
-    #     for filename in os.listdir(folder_path):
-    #         # Check if the search word is in the filename
-    #         if "model_cc.md" in filename.lower():
-
-    #             # If it is, load the yaml
-
-    #             with open(folder_path + filename, 'r') as file:
-    #                 model_cc_yaml = yaml.safe_load(file)
-
-    #             for key, value in model_cc_yaml['obligations_for_providers_of_gpai_models']:
-    #                 if not value:
-    #                     msg = (f"Because of the model represented by {filename}, this GPAI fails the transparency requirements under Article 53.")
-
-    # # If the project is a GPAI model with systematic risk, check that is has additionally met all the requirements for such systems: 
-
-    # if gpai_model_systematic_risk:
-
-    # # Do this by examining the Project CC
-
-    #     for key, value in project_cc_yaml['gpai_obligations_for_systemic_risk_models']:
-    #         if not value:
-    #             msg = ("GPAI model with systematic risk fails the transparency requirements under Article 55.")
-
-    # # Do this by examining any and all Model CCs too
-
-    #     for filename in os.listdir(folder_path):
-    #         # Check if the search word is in the filename
-    #         if "model_cc.md" in filename.lower():
-
-    #             # If it is, load the yaml
-
-    #             with open(folder_path + filename, 'r') as file:
-    #                 model_cc_yaml = yaml.safe_load(file)
-
-    #             for key, value in model_cc_yaml['obligations_for_providers_of_gpai_models_with_systemic_risk']:
-    #                 if not value:
-    #                     msg = (f"Because of the model represented by {filename}, this GPAI model with systematic risk fails the transparency requirements under Article 55.")
-
+    return dispositive_variables
 
 

project_cc.yaml

@@ -2,13 +2,16 @@
 # Information related to high-level characteristics of AI project, including the role of the operator, their location, and where the output is used
 
 operator_details:
-  provider: # Art. 2
+  provider: 
+    article: "Art. 2"
     verbose: 'The operator of this AI project is a natural or legal person, public authority, agency or other body that develops an AI project or a general-purpose AI model or that has an AI system or a general-purpose AI model developed and places it on the market or puts the AI system into service under its own name or trademark, whether for payment or free of charge'
     value: !!bool false 
-  eu_located: # Art. 2
+  eu_located: 
+    article: 'Art. 2'
     verbose: 'AI project operator has its place of establishment or location within the Union'
     value: !!bool True  
-  output_used: # Art. 2
+  output_used:
+    article: 'Art. 2'
     verbose: 'The output produced by the AI project is used in the Union'
     value: !!bool false 
 
@@ -458,7 +461,7 @@ transparency_obligations:
 
 # Information related to the Act's requirements for GPAI models
 
-gpai_model_provider_obligations:
+gpai_model_obligations:
   documentation:
     intended_uses: # Art. 53(1)(a); Annex XI(1)(1)(a)
       verbose: 'The provider has drawn up and will keep up-to-date technical documentation of the model that include a general description of the model includes a description of the tasks that the model is intended to perform and the type and nature of AI systems in which it can be integrated'
@@ -539,7 +542,7 @@ gpai_model_provider_obligations:
 
 # Information related to the Act's requirements for GPAI models with systematic risk 
 
-obligations_for_gpai_models_with_systemic_risk:
+gpai_models_with_systemic_risk_obligations:
   notification: # Art 52(1)
     verbose: 'Within two weeks of it being known that the AI project should be classified as a GPAI model with systemtic ris, tkhe Commission was notified and provided with the information that supports this finding'
   evaluation: # Art. 55(1)(a)
@@ -564,7 +567,8 @@ obligations_for_gpai_models_with_systemic_risk:
     verbose: 'The provider has drawn up and will keep up-to-date technical documentation of the model that includes, where applicable, a detailed description of the measures put in place for the purpose of conducting internal and/or external adversarial testing (e.g. red teaming), model adaptations, including alignment and fine-tuning.'
     value: !!bool false 
   documentation_architecture: 
-    verbose: ''The provider has drawn up and will keep up-to-date technical documentation of the model that includes, where applicable, a detailed description of the system architecture explaining how software components build or feed into each other and integrate into the overall processing.'
+    verbose: 'The provider has drawn up and will keep up-to-date technical documentation of the model that includes, where applicable, a detailed description of the system architecture explaining how software components build or feed into each other and integrate into the overall processing.'
+    value: !!bool false
 
 additional_provider_obligations: # apply these only if operator == provider and ai_project_type == high_risk_ai_system
   contact: # Article 16 (b)

utils.py

@@ -97,6 +97,7 @@ def check_excepted(project_cc_yaml):
     else:
         return False 
 
+# TODO update function
 def check_prohibited(dispositive_variables, project_cc_yaml):
 
     ai_system = project_variables['ai_project_type']['ai_system']