ignore files in data collection

project/DataCollectionPipeline.ipynb

@@ -3476,6 +3476,8 @@
     "                        subdirectory is not None and\n",
     "                        'http' not in subdirectory and\n",
     "                        '#' not in subdirectory and\n",
+    "                        '.zip' not in subdirectory and\n",
+    "                        '.pdf' not in subdirectory and\n",
     "                        mongoCollection.find_one({\"link\": newLink}) is None and\n",
     "                        newLink not in links\n",
     "                    ):\n",