{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "fc652877-6e04-43fb-b4ce-0a08a07f7b99", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into 'AutoAWQ'...\n", "remote: Enumerating objects: 3588, done.\u001b[K\n", "remote: Counting objects: 100% (842/842), done.\u001b[K\n", "remote: Compressing objects: 100% (303/303), done.\u001b[K\n", "remote: Total 3588 (delta 724), reused 561 (delta 539), pack-reused 2746 (from 3)\u001b[K\n", "Receiving objects: 100% (3588/3588), 7.16 MiB | 29.69 MiB/s, done.\n", "Resolving deltas: 100% (2276/2276), done.\n", "Obtaining file:///workspace/AutoAWQ\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (2.4.1+cu124)\n", "Requirement already satisfied: triton in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (3.0.0)\n", "Requirement already satisfied: transformers>=4.45.0 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (4.49.0)\n", "Requirement already satisfied: tokenizers>=0.12.1 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.21.0)\n", "Requirement already satisfied: typing_extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (4.12.2)\n", "Requirement already satisfied: accelerate in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (1.4.0)\n", "Requirement already satisfied: datasets>=2.20 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (3.3.2)\n", "Requirement already satisfied: zstandard in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.23.0)\n", "Requirement already satisfied: huggingface_hub>=0.26.5 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.29.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.13.1)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (1.26.4)\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (19.0.1)\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (0.3.8)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (2.2.3)\n", "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (2.32.3)\n", "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (4.67.1)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.5.0)\n", "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (0.70.16)\n", "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets>=2.20->autoawq==0.2.8) (2024.2.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.11.12)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (24.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers>=4.45.0->autoawq==0.2.8) (2024.11.6)\n", "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.11/dist-packages (from transformers>=4.45.0->autoawq==0.2.8) (0.5.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from accelerate->autoawq==0.2.8) (6.0.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (3.1.3)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n", "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (9.1.0.70)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.4.2.65 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.2.65)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.2.0.44 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (11.2.0.44)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.5.119 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (10.3.5.119)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.6.0.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (11.6.0.99)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.3.0.142 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.3.0.142)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (2.20.5)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (2.4.6)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.3.2)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (24.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (6.1.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (0.3.0)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.18.3)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (2024.8.30)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch->autoawq==0.2.8) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2025.1)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2025.1)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/dist-packages (from sympy->torch->autoawq==0.2.8) (1.3.0)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.20->autoawq==0.2.8) (1.16.0)\n", "Installing collected packages: autoawq\n", " Attempting uninstall: autoawq\n", " Found existing installation: autoawq 0.2.7.post3\n", " Uninstalling autoawq-0.2.7.post3:\n", " Successfully uninstalled autoawq-0.2.7.post3\n", "\u001b[33m DEPRECATION: Legacy editable install of autoawq==0.2.8 from file:///workspace/AutoAWQ (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457\u001b[0m\u001b[33m\n", "\u001b[0m Running setup.py develop for autoawq\n", "Successfully installed autoawq-0.2.8\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (4.49.0)\n", "Collecting transformers\n", " Downloading transformers-4.51.3-py3-none-any.whl.metadata (38 kB)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from transformers) (3.13.1)\n", "Collecting huggingface-hub<1.0,>=0.30.0 (from transformers)\n", " Downloading huggingface_hub-0.30.2-py3-none-any.whl.metadata (13 kB)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from transformers) (24.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (2024.11.6)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.21.0)\n", "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.5.2)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.11/dist-packages (from transformers) (4.67.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (2024.2.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2024.8.30)\n", "Downloading transformers-4.51.3-py3-none-any.whl (10.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.4/10.4 MB\u001b[0m \u001b[31m168.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading huggingface_hub-0.30.2-py3-none-any.whl (481 kB)\n", "Installing collected packages: huggingface-hub, transformers\n", " Attempting uninstall: huggingface-hub\n", " Found existing installation: huggingface-hub 0.29.1\n", " Uninstalling huggingface-hub-0.29.1:\n", " Successfully uninstalled huggingface-hub-0.29.1\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.49.0\n", " Uninstalling transformers-4.49.0:\n", " Successfully uninstalled transformers-4.49.0\n", "Successfully installed huggingface-hub-0.30.2 transformers-4.51.3\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n" ] } ], "source": [ "!git clone -b llama4 https://github.com/kIshizaki-sci/AutoAWQ.git\n", "!pip install -e ./AutoAWQ\n", "!pip install -U transformers" ] }, { "cell_type": "code", "execution_count": 1, "id": "846f4a3e-5960-4b52-8c79-74eee1f72468", "metadata": {}, "outputs": [], "source": [ "from awq import AutoAWQForCausalLM\n", "import torch\n", "import transformers\n", "from transformers import AutoProcessor" ] }, { "cell_type": "code", "execution_count": 2, "id": "fc58d0e4-e98a-4116-b27c-87ef1adf3620", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch version : 2.4.1+cu124\n", "transformers version : 4.51.3\n" ] } ], "source": [ "print('torch version : ', torch.__version__)\n", "print('transformers version : ', transformers.__version__)" ] }, { "cell_type": "code", "execution_count": 3, "id": "47a07cd8-7874-4b57-9805-ed1f5b0dbd17", "metadata": {}, "outputs": [], "source": [ "quant_path = 'kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ'" ] }, { "cell_type": "code", "execution_count": 4, "id": "adaaeee2-c985-430c-8aca-d53821418536", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/workspace/AutoAWQ/awq/models/llama4.py:312: UserWarning: Multimodal input has not been implemented in Llama4AWQForConditionalGeneration yet.\n", " warnings.warn(\"Multimodal input has not been implemented in Llama4AWQForConditionalGeneration yet.\", UserWarning)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "26119589fe994dd494eb7717c011fcc0", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Fetching 25 files: 0%| | 0/25 [00:00