Allegro_Music_Transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Allegro Music Transformer (ver. 5.0)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2023\n",
        "\n",
        "***"
      ],
      "metadata": {
        "id": "gpy3qsulqHa5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GPU CHECK)"
      ],
      "metadata": {
        "id": "W_So4w8fqPGL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X3rABEpKCO02",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title NVIDIA GPU check\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (SETUP ENVIRONMENT)"
      ],
      "metadata": {
        "id": "C0XxnXGFqVyh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vK40g6V_BTNj",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install dependencies\n",
        "!git clone --depth 1 https://github.com/asigalov61/Allegro-Music-Transformer\n",
        "!pip install huggingface_hub\n",
        "!pip install torch\n",
        "!pip install einops\n",
        "!pip install torch-summary\n",
        "!pip install tqdm\n",
        "!pip install matplotlib\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works\n",
        "!pip install midi2audio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DzCOZU_gBiQV",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Import modules\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading core Allegro Music Transformer modules...')\n",
        "\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import secrets\n",
        "import statistics\n",
        "from time import time\n",
        "import tqdm\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading main Allegro Music Transformer modules...')\n",
        "import torch\n",
        "\n",
        "%cd /content/Allegro-Music-Transformer\n",
        "\n",
        "import TMIDIX\n",
        "from x_transformer import *\n",
        "\n",
        "%cd /content/\n",
        "print('=' * 70)\n",
        "print('Loading aux Allegro Music Transformer modules...')\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from torchsummary import summary\n",
        "from sklearn import metrics\n",
        "\n",
        "from midi2audio import FluidSynth\n",
        "from IPython.display import Audio, display\n",
        "\n",
        "from huggingface_hub import hf_hub_download\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('Enjoy! :)')\n",
        "print('=' * 70)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eI3aQtHzqSnp"
      },
      "source": [
        "# (LOAD MODEL)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Choose one and do not forget to restart colab runtime if switching between models"
      ],
      "metadata": {
        "id": "BhMzyohMliMB"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rDquonbXC2je",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Load Allegro Music Transformer Tiny Model (FAST)\n",
        "\n",
        "#@markdown Very fast model, 16 layers, 225k MIDIs training corpus\n",
        "\n",
        "full_path_to_model_checkpoint = \"/content/Allegro-Music-Transformer/Models/Tiny/Allegro_Music_Transformer_Tiny_Trained_Model_80000_steps_0.9457_loss_0.7443_acc.pth\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown Model precision option\n",
        "\n",
        "model_precision = \"bfloat16\" # @param [\"bfloat16\", \"float16\", \"float32\"]\n",
        "\n",
        "#@markdown bfloat16 == Third precision/triple speed (if supported, otherwise the model will default to float16)\n",
        "\n",
        "#@markdown float16 == Half precision/double speed\n",
        "\n",
        "#@markdown float32 == Full precision/normal speed\n",
        "\n",
        "plot_tokens_embeddings = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading Allegro Music Transformer Tiny Pre-Trained Model...')\n",
        "print('Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "if os.path.isfile(full_path_to_model_checkpoint):\n",
        "  print('Model already exists...')\n",
        "\n",
        "else:\n",
        "  hf_hub_download(repo_id='asigalov61/Allegro-Music-Transformer',\n",
        "                  filename='Allegro_Music_Transformer_Tiny_Trained_Model_80000_steps_0.9457_loss_0.7443_acc.pth',\n",
        "                  local_dir='/content/Allegro-Music-Transformer/Models/Tiny/',\n",
        "                  local_dir_use_symlinks=False)\n",
        "print('=' * 70)\n",
        "print('Instantiating model...')\n",
        "\n",
        "torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul\n",
        "torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn\n",
        "device_type = 'cuda'\n",
        "\n",
        "if model_precision == 'bfloat166' and torch.cuda.is_bf16_supported():\n",
        "  dtype = 'bfloat16'\n",
        "else:\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float16':\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float32':\n",
        "  dtype = 'float32'\n",
        "\n",
        "ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]\n",
        "ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)\n",
        "\n",
        "SEQ_LEN = 2048\n",
        "\n",
        "# instantiate the model\n",
        "\n",
        "model = TransformerWrapper(\n",
        "    num_tokens = 3088,\n",
        "    max_seq_len = SEQ_LEN,\n",
        "    attn_layers = Decoder(dim = 1024, depth = 16, heads = 8, attn_flash=True)\n",
        ")\n",
        "\n",
        "model = AutoregressiveWrapper(model)\n",
        "\n",
        "model = torch.nn.DataParallel(model)\n",
        "\n",
        "model.cuda()\n",
        "print('=' * 70)\n",
        "\n",
        "print('Loading model checkpoint...')\n",
        "\n",
        "model.load_state_dict(torch.load(full_path_to_model_checkpoint))\n",
        "print('=' * 70)\n",
        "\n",
        "model.eval()\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Model will use', dtype, 'precision...')\n",
        "print('=' * 70)\n",
        "\n",
        "# Model stats\n",
        "print('Model summary...')\n",
        "summary(model)\n",
        "\n",
        "# Plot Token Embeddings\n",
        "\n",
        "if plot_tokens_embeddings:\n",
        "\n",
        "  tok_emb = model.module.net.token_emb.emb.weight.detach().cpu().tolist()\n",
        "\n",
        "  cos_sim = metrics.pairwise_distances(\n",
        "    tok_emb, metric='cosine'\n",
        "  )\n",
        "  plt.figure(figsize=(7, 7))\n",
        "  plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"nearest\")\n",
        "  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]\n",
        "  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "  plt.xlabel(\"Position\")\n",
        "  plt.ylabel(\"Position\")\n",
        "  plt.tight_layout()\n",
        "  plt.plot()\n",
        "  plt.savefig(\"/content/Allegro-Music-Transformer-Tiny-Tokens-Embeddings-Plot.png\", bbox_inches=\"tight\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load Allegro Music Transformer Small Model (BEST)\n",
        "\n",
        "#@markdown Fast model, 32 layers, 225k MIDIs training corpus\n",
        "\n",
        "full_path_to_model_checkpoint = \"/content/Allegro-Music-Transformer/Models/Small/Allegro_Music_Transformer_Small_Trained_Model_56000_steps_0.9399_loss_0.7374_acc.pth\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown Model precision option\n",
        "\n",
        "model_precision = \"bfloat16\" # @param [\"bfloat16\", \"float16\", \"float32\"]\n",
        "\n",
        "#@markdown bfloat16 == Third precision/triple speed (if supported, otherwise the model will default to float16)\n",
        "\n",
        "#@markdown float16 == Half precision/double speed\n",
        "\n",
        "#@markdown float32 == Full precision/normal speed\n",
        "\n",
        "plot_tokens_embeddings = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading Allegro Music Transformer Small Pre-Trained Model...')\n",
        "print('Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "if os.path.isfile(full_path_to_model_checkpoint):\n",
        "  print('Model already exists...')\n",
        "\n",
        "else:\n",
        "  hf_hub_download(repo_id='asigalov61/Allegro-Music-Transformer',\n",
        "                  filename='Allegro_Music_Transformer_Small_Trained_Model_56000_steps_0.9399_loss_0.7374_acc.pth',\n",
        "                  local_dir='/content/Allegro-Music-Transformer/Models/Small/',\n",
        "                  local_dir_use_symlinks=False)\n",
        "print('=' * 70)\n",
        "print('Instantiating model...')\n",
        "\n",
        "torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul\n",
        "torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn\n",
        "device_type = 'cuda'\n",
        "\n",
        "if model_precision == 'bfloat16' and torch.cuda.is_bf16_supported():\n",
        "  dtype = 'bfloat16'\n",
        "else:\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float16':\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float32':\n",
        "  dtype = 'float32'\n",
        "\n",
        "ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]\n",
        "ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)\n",
        "\n",
        "SEQ_LEN = 2048\n",
        "\n",
        "# instantiate the model\n",
        "\n",
        "model = TransformerWrapper(\n",
        "    num_tokens = 3088,\n",
        "    max_seq_len = SEQ_LEN,\n",
        "    attn_layers = Decoder(dim = 1024, depth = 32, heads = 8, attn_flash=True)\n",
        ")\n",
        "\n",
        "model = AutoregressiveWrapper(model)\n",
        "\n",
        "model = torch.nn.DataParallel(model)\n",
        "\n",
        "model.cuda()\n",
        "print('=' * 70)\n",
        "\n",
        "print('Loading model checkpoint...')\n",
        "\n",
        "model.load_state_dict(torch.load(full_path_to_model_checkpoint))\n",
        "print('=' * 70)\n",
        "\n",
        "model.eval()\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Model will use', dtype, 'precision...')\n",
        "print('=' * 70)\n",
        "\n",
        "# Model stats\n",
        "print('Model summary...')\n",
        "summary(model)\n",
        "\n",
        "# Plot Token Embeddings\n",
        "\n",
        "if plot_tokens_embeddings:\n",
        "\n",
        "  tok_emb = model.module.net.token_emb.emb.weight.detach().cpu().tolist()\n",
        "\n",
        "  cos_sim = metrics.pairwise_distances(\n",
        "    tok_emb, metric='cosine'\n",
        "  )\n",
        "  plt.figure(figsize=(7, 7))\n",
        "  plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"nearest\")\n",
        "  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]\n",
        "  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "  plt.xlabel(\"Position\")\n",
        "  plt.ylabel(\"Position\")\n",
        "  plt.tight_layout()\n",
        "  plt.plot()\n",
        "  plt.savefig(\"/content/Allegro-Music-Transformer-Small-Tokens-Embeddings-Plot.png\", bbox_inches=\"tight\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "r2CBWPbalnVQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GENERATE)"
      ],
      "metadata": {
        "id": "7xNyANjZsCOi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (IMPROV)"
      ],
      "metadata": {
        "id": "ZrEQDD5OuYIb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Standard Improv Generator\n",
        "\n",
        "#@markdown Improv settings\n",
        "\n",
        "#@markdown NOTE: The improv settings below are just the strong suggestions for the model, not the requirements.\n",
        "\n",
        "#@markdown Some settings combinations may not work well.\n",
        "\n",
        "first_note_instrument = \"Piano\" #@param [\"Piano\", \"Guitar\", \"Bass\", \"Violin\", \"Cello\", \"Harp\", \"Trumpet\", \"Sax\", \"Flute\", \"Choir\", \"Organ\"]\n",
        "add_drums = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown Generation settings\n",
        "\n",
        "number_of_tokens_tp_generate = 300 # @param {type:\"slider\", min:30, max:2048, step:3}\n",
        "number_of_batches_to_generate = 4 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "temperature = 0.9 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
        "\n",
        "#@markdown Other settings\n",
        "\n",
        "render_MIDI_to_audio = True # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Allegro Music Transformer Standard Improv Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "if add_drums:\n",
        "  drumsp = 3074 # Yes\n",
        "else:\n",
        "  drumsp = 3073 # No\n",
        "\n",
        "instruments_list = [\"Piano\", \"Guitar\", \"Bass\", \"Violin\", \"Cello\", \"Harp\", \"Trumpet\", \"Sax\", \"Flute\", 'Drums', \"Choir\", \"Organ\"]\n",
        "first_note_instrument_number = instruments_list.index(first_note_instrument)\n",
        "\n",
        "outy = [3087, drumsp, 3075+first_note_instrument_number]\n",
        "\n",
        "print('Selected Improv sequence:')\n",
        "print(outy)\n",
        "print('=' * 70)\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "inp = [outy] * number_of_batches_to_generate\n",
        "\n",
        "inp = torch.LongTensor(inp).cuda()\n",
        "\n",
        "with ctx:\n",
        "  out = model.module.generate(inp,\n",
        "                        number_of_tokens_tp_generate,\n",
        "                        temperature=temperature,\n",
        "                        return_prime=True,\n",
        "                        verbose=True)\n",
        "\n",
        "out0 = out.tolist()\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "#======================================================================\n",
        "\n",
        "print('Rendering results...')\n",
        "\n",
        "for i in range(number_of_batches_to_generate):\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Batch #', i)\n",
        "  print('=' * 70)\n",
        "\n",
        "  out1 = out0[i]\n",
        "\n",
        "  print('Sample INTs', out1[:12])\n",
        "  print('=' * 70)\n",
        "\n",
        "  if len(out1) != 0:\n",
        "\n",
        "      song = out1\n",
        "      song_f = []\n",
        "\n",
        "      time = 0\n",
        "      dur = 0\n",
        "      vel = 90\n",
        "      pitch = 0\n",
        "      channel = 0\n",
        "\n",
        "      for ss in song:\n",
        "\n",
        "        if ss > 0 and ss < 256:\n",
        "\n",
        "            time += ss * 8\n",
        "\n",
        "        if ss >= 256 and ss < 1280:\n",
        "\n",
        "            dur = ((ss-256) // 8) * 32\n",
        "            vel = (((ss-256) % 8)+1) * 15\n",
        "\n",
        "        if ss >= 1280 and ss < 2816:\n",
        "            channel = (ss-1280) // 128\n",
        "            pitch = (ss-1280) % 128\n",
        "\n",
        "            song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                                output_signature = 'Allegro Music Transformer',\n",
        "                                                                output_file_name = '/content/Allegro-Music-Transformer-Composition_'+str(i),\n",
        "                                                                track_name='Project Los Angeles',\n",
        "                                                                list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 65, 73, 0, 53, 19, 0, 0, 0, 0]\n",
        "                                                                )\n",
        "\n",
        "\n",
        "      print('=' * 70)\n",
        "      print('Displaying resulting composition...')\n",
        "      print('=' * 70)\n",
        "\n",
        "      fname = '/content/Allegro-Music-Transformer-Composition_'+str(i)\n",
        "\n",
        "      x = []\n",
        "      y =[]\n",
        "      c = []\n",
        "\n",
        "      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']\n",
        "\n",
        "      for s in song_f:\n",
        "        x.append(s[1] / 1000)\n",
        "        y.append(s[4])\n",
        "        c.append(colors[s[3]])\n",
        "\n",
        "      if render_MIDI_to_audio:\n",
        "        FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "        display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "      plt.figure(figsize=(14,5))\n",
        "      ax=plt.axes(title=fname)\n",
        "      ax.set_facecolor('black')\n",
        "\n",
        "      plt.scatter(x,y, c=c)\n",
        "      plt.xlabel(\"Time\")\n",
        "      plt.ylabel(\"Pitch\")\n",
        "      plt.show()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Jwxz-eaF0K1y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (CUSTOM MIDI)"
      ],
      "metadata": {
        "id": "Gt03VtO6uKkb"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4QXbFLsKqSnt",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Load Seed MIDI\n",
        "\n",
        "#@markdown Press play button to to upload your own seed MIDI or to load one of the provided sample seed MIDIs from the dropdown list below\n",
        "\n",
        "select_seed_MIDI = \"Upload your own custom MIDI\" #@param [\"Upload your own custom MIDI\", \"Allegro-Music-Transformer-Piano-Seed-1\", \"Allegro-Music-Transformer-Piano-Seed-2\", \"Allegro-Music-Transformer-Piano-Seed-3\", \"Allegro-Music-Transformer-Piano-Seed-4\", \"Allegro-Music-Transformer-Piano-Seed-5\", \"Allegro-Music-Transformer-MI-Seed-1\", \"Allegro-Music-Transformer-MI-Seed-2\", \"Allegro-Music-Transformer-MI-Seed-3\", \"Allegro-Music-Transformer-MI-Seed-4\", \"Allegro-Music-Transformer-MI-Seed-5\"]\n",
        "render_MIDI_to_audio = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Allegro Music Transformer Seed MIDI Loader')\n",
        "print('=' * 70)\n",
        "\n",
        "f = ''\n",
        "\n",
        "if select_seed_MIDI != \"Upload your own custom MIDI\":\n",
        "  print('Loading seed MIDI...')\n",
        "  f = '/content/Allegro-Music-Transformer/Seeds/'+select_seed_MIDI+'.mid'\n",
        "  score = TMIDIX.midi2single_track_ms_score(open(f, 'rb').read(), recalculate_channels=False)\n",
        "\n",
        "else:\n",
        "  print('Upload your own custom MIDI...')\n",
        "  print('=' * 70)\n",
        "  uploaded_MIDI = files.upload()\n",
        "  if list(uploaded_MIDI.keys()):\n",
        "    score = TMIDIX.midi2single_track_ms_score(list(uploaded_MIDI.values())[0], recalculate_channels=False)\n",
        "    f = list(uploaded_MIDI.keys())[0]\n",
        "\n",
        "if f != '':\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('File:', f)\n",
        "  print('=' * 70)\n",
        "\n",
        "  #=======================================================\n",
        "  # START PROCESSING\n",
        "\n",
        "  melody_chords_f = []\n",
        "\n",
        "  # INSTRUMENTS CONVERSION CYCLE\n",
        "  events_matrix = []\n",
        "  itrack = 1\n",
        "  patches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "\n",
        "  patch_map = [\n",
        "              [0, 1, 2, 3, 4, 5, 6, 7], # Piano\n",
        "              [24, 25, 26, 27, 28, 29, 30], # Guitar\n",
        "              [32, 33, 34, 35, 36, 37, 38, 39], # Bass\n",
        "              [40, 41], # Violin\n",
        "              [42, 43], # Cello\n",
        "              [46], # Harp\n",
        "              [56, 57, 58, 59, 60], # Trumpet\n",
        "              [64, 65, 66, 67, 68, 69, 70, 71], # Sax\n",
        "              [72, 73, 74, 75, 76, 77, 78], # Flute\n",
        "              [-1], # Drums\n",
        "              [52, 53], # Choir\n",
        "              [16, 17, 18, 19, 20] # Organ\n",
        "              ]\n",
        "\n",
        "  while itrack < len(score):\n",
        "      for event in score[itrack]:\n",
        "          if event[0] == 'note' or event[0] == 'patch_change':\n",
        "              events_matrix.append(event)\n",
        "      itrack += 1\n",
        "\n",
        "  events_matrix.sort(key=lambda x: x[1])\n",
        "\n",
        "  events_matrix1 = []\n",
        "\n",
        "  for event in events_matrix:\n",
        "          if event[0] == 'patch_change':\n",
        "              patches[event[2]] = event[3]\n",
        "\n",
        "          if event[0] == 'note':\n",
        "              event.extend([patches[event[3]]])\n",
        "              once = False\n",
        "\n",
        "              for p in patch_map:\n",
        "                  if event[6] in p and event[3] != 9: # Except the drums\n",
        "                      event[3] = patch_map.index(p)\n",
        "                      once = True\n",
        "\n",
        "              if not once and event[3] != 9: # Except the drums\n",
        "                  event[3] = 15 # All other instruments/patches channel\n",
        "                  event[5] = max(80, event[5])\n",
        "\n",
        "              if event[3] < 12: # We won't write chans 12-16 for now...\n",
        "                  events_matrix1.append(event)\n",
        "\n",
        "  if len(events_matrix1) > 0:\n",
        "\n",
        "\n",
        "      #=======================================================\n",
        "      # PRE-PROCESSING\n",
        "\n",
        "      # checking number of instruments in a composition\n",
        "      instruments_list_without_drums = list(set([y[3] for y in events_matrix1 if y[3] != 9]))\n",
        "\n",
        "      if len(events_matrix1) > 0 and len(instruments_list_without_drums) > 0:\n",
        "\n",
        "        # recalculating timings\n",
        "        for e in events_matrix1:\n",
        "            e[1] = int(e[1] / 8) # Max 2 seconds for start-times\n",
        "            e[2] = int(e[2] / 32) # Max 4 seconds for durations\n",
        "\n",
        "        # Sorting by pitch, then by start-time\n",
        "        events_matrix1.sort(key=lambda x: x[4], reverse=True)\n",
        "        events_matrix1.sort(key=lambda x: x[1])\n",
        "\n",
        "        #=======================================================\n",
        "        # FINAL PRE-PROCESSING\n",
        "\n",
        "        melody_chords = []\n",
        "\n",
        "        pe = events_matrix1[0]\n",
        "\n",
        "        for e in events_matrix1:\n",
        "\n",
        "            # Cliping all values...\n",
        "            time = max(0, min(255, e[1]-pe[1]))\n",
        "            dur = max(1, min(127, e[2]))\n",
        "            cha = max(0, min(11, e[3]))\n",
        "            ptc = max(1, min(127, e[4]))\n",
        "\n",
        "            # Calculating octo-velocity\n",
        "            vel = max(8, min(127, e[5]))\n",
        "            velocity = round(vel / 15)-1\n",
        "\n",
        "            # Writing final note\n",
        "            melody_chords.append([time, dur, cha, ptc, velocity])\n",
        "\n",
        "            pe = e\n",
        "\n",
        "        times = [y[0] for y in melody_chords[12:]]\n",
        "        avg_time = sum(times) / len(times)\n",
        "\n",
        "        times_list = list(set(times))\n",
        "\n",
        "        mode_dur = statistics.mode([y[1] for y in melody_chords if y[2] != 9])\n",
        "\n",
        "        instruments_list = list(set([y[2] for y in melody_chords]))\n",
        "        num_instr = len(instruments_list)\n",
        "\n",
        "        #=======================================================\n",
        "\n",
        "        # TOTAL DICTIONARY SIZE 3087+1=3088\n",
        "\n",
        "        #=======================================================\n",
        "        # MAIN PROCESSING CYCLE\n",
        "        #=======================================================\n",
        "\n",
        "        chords_count = 0\n",
        "\n",
        "        melody_chords_f.extend([2816]) # Zero chords count\n",
        "\n",
        "        if melody_chords[0][0] == 0:\n",
        "          melody_chords_f.extend([0]) # Zero time, if present\n",
        "\n",
        "        notes_counter = 0\n",
        "        chords_counter = 0\n",
        "\n",
        "        for m in melody_chords:\n",
        "\n",
        "          time = m[0]\n",
        "\n",
        "          # Chords counter token\n",
        "          if chords_count % 50 == 0 and chords_count != 0 and time != 0:\n",
        "            melody_chords_f.extend([2816+min(255, ((chords_count // 50)))])\n",
        "\n",
        "          if time != 0:\n",
        "            chords_count += 1\n",
        "\n",
        "          # WRITING EACH NOTE HERE\n",
        "          dur_vel = (m[1] * 8) + m[4]\n",
        "          cha_ptc = (m[2] * 128) + m[3]\n",
        "\n",
        "          if time != 0:\n",
        "              melody_chords_f.extend([time, dur_vel+256, cha_ptc+1280])\n",
        "              chords_counter += 1\n",
        "\n",
        "          else:\n",
        "              melody_chords_f.extend([dur_vel+256, cha_ptc+1280])\n",
        "\n",
        "          notes_counter += 1\n",
        "\n",
        "  #=======================================================\n",
        "\n",
        "  song = melody_chords_f\n",
        "\n",
        "  song_f = []\n",
        "\n",
        "  time = 0\n",
        "  dur = 0\n",
        "  vel = 90\n",
        "  pitch = 0\n",
        "  channel = 0\n",
        "\n",
        "  for ss in song:\n",
        "\n",
        "    if ss > 0 and ss < 256:\n",
        "\n",
        "        time += ss * 8\n",
        "\n",
        "    if ss >= 256 and ss < 1280:\n",
        "\n",
        "        dur = ((ss-256) // 8) * 32\n",
        "        vel = (((ss-256) % 8)+1) * 15\n",
        "\n",
        "    if ss >= 1280 and ss < 2816:\n",
        "        channel = (ss-1280) // 128\n",
        "        pitch = (ss-1280) % 128\n",
        "\n",
        "        song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "  detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                            output_signature = 'Allegro Music Transformer',\n",
        "                                                            output_file_name = '/content/Allegro-Music-Transformer-Seed-Composition',\n",
        "                                                            track_name='Project Los Angeles',\n",
        "                                                            list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 65, 73, 0, 53, 19, 0, 0, 0, 0]\n",
        "                                                            )\n",
        "\n",
        "  #=======================================================\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Composition stats:')\n",
        "  print('Composition has', notes_counter, 'notes')\n",
        "  print('Composition has', chords_counter, 'chords')\n",
        "  print('Composition has', len(melody_chords_f), 'tokens')\n",
        "  print('=' * 70)\n",
        "\n",
        "  fname = '/content/Allegro-Music-Transformer-Seed-Composition'\n",
        "\n",
        "  x = []\n",
        "  y =[]\n",
        "  c = []\n",
        "\n",
        "  colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']\n",
        "\n",
        "  for s in song_f:\n",
        "    x.append(s[1] / 1000)\n",
        "    y.append(s[4])\n",
        "    c.append(colors[s[3]])\n",
        "\n",
        "  if render_MIDI_to_audio:\n",
        "    FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "    display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "  plt.figure(figsize=(14,5))\n",
        "  ax=plt.axes(title=fname)\n",
        "  ax.set_facecolor('black')\n",
        "\n",
        "  plt.scatter(x,y, c=c)\n",
        "  plt.xlabel(\"Time\")\n",
        "  plt.ylabel(\"Pitch\")\n",
        "  plt.show()\n",
        "\n",
        "else:\n",
        "  print('=' * 70)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (CONTINUATION)"
      ],
      "metadata": {
        "id": "1MHGQzg81_jT"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dkvXYwR_qSnx",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Standard Continuation Generator\n",
        "\n",
        "#@markdown Generation settings\n",
        "\n",
        "try_to_generate_outro = False #@param {type:\"boolean\"}\n",
        "number_of_prime_tokens = 300 #@param {type:\"slider\", min:3, max:2046, step:3}\n",
        "number_of_tokens_to_generate = 300 #@param {type:\"slider\", min:30, max:2046, step:30}\n",
        "number_of_batches_to_generate = 4 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "temperature = 0.9 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
        "\n",
        "#@markdown Other settings\n",
        "\n",
        "include_prime_tokens_in_generated_output = True #@param {type:\"boolean\"}\n",
        "allow_model_to_stop_generation_if_needed = False #@param {type:\"boolean\"}\n",
        "render_MIDI_to_audio = True # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Allegro Music Transformer Standard Continuation Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "if allow_model_to_stop_generation_if_needed:\n",
        "  min_stop_token = 3087\n",
        "else:\n",
        "  min_stop_token = None\n",
        "\n",
        "outy = melody_chords_f[:number_of_prime_tokens]\n",
        "\n",
        "if try_to_generate_outro:\n",
        "  outy.extend([3072])\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "inp = [outy] * number_of_batches_to_generate\n",
        "\n",
        "inp = torch.LongTensor(inp).cuda()\n",
        "\n",
        "with ctx:\n",
        "  out = model.module.generate(inp,\n",
        "                        number_of_tokens_to_generate,\n",
        "                        temperature=temperature,\n",
        "                        return_prime=include_prime_tokens_in_generated_output,\n",
        "                        eos_token=min_stop_token,\n",
        "                        verbose=True)\n",
        "\n",
        "out0 = out.tolist()\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "#======================================================================\n",
        "print('Rendering results...')\n",
        "\n",
        "for i in range(number_of_batches_to_generate):\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Batch #', i)\n",
        "  print('=' * 70)\n",
        "\n",
        "  out1 = out0[i]\n",
        "\n",
        "  print('Sample INTs', out1[:12])\n",
        "  print('=' * 70)\n",
        "\n",
        "  if len(out) != 0:\n",
        "\n",
        "      song = out1\n",
        "      song_f = []\n",
        "\n",
        "      time = 0\n",
        "      dur = 0\n",
        "      vel = 90\n",
        "      pitch = 0\n",
        "      channel = 0\n",
        "\n",
        "      for ss in song:\n",
        "\n",
        "        if ss > 0 and ss < 256:\n",
        "\n",
        "            time += ss * 8\n",
        "\n",
        "        if ss >= 256 and ss < 1280:\n",
        "\n",
        "            dur = ((ss-256) // 8) * 32\n",
        "            vel = (((ss-256) % 8)+1) * 15\n",
        "\n",
        "        if ss >= 1280 and ss < 2816:\n",
        "            channel = (ss-1280) // 128\n",
        "            pitch = (ss-1280) % 128\n",
        "\n",
        "            song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                                output_signature = 'Allegro Music Transformer',\n",
        "                                                                output_file_name = '/content/Allegro-Music-Transformer-Composition_'+str(i),\n",
        "                                                                track_name='Project Los Angeles',\n",
        "                                                                list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 65, 73, 0, 53, 19, 0, 0, 0, 0]\n",
        "                                                                )\n",
        "      print('=' * 70)\n",
        "      print('Displaying resulting composition...')\n",
        "      print('=' * 70)\n",
        "\n",
        "      fname = '/content/Allegro-Music-Transformer-Composition_'+str(i)\n",
        "\n",
        "      x = []\n",
        "      y =[]\n",
        "      c = []\n",
        "\n",
        "      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']\n",
        "\n",
        "      for s in song_f:\n",
        "        x.append(s[1] / 1000)\n",
        "        y.append(s[4])\n",
        "        c.append(colors[s[3]])\n",
        "\n",
        "      if render_MIDI_to_audio:\n",
        "        FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "        display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "      plt.figure(figsize=(14,5))\n",
        "      ax=plt.axes(title=fname)\n",
        "      ax.set_facecolor('black')\n",
        "\n",
        "      plt.scatter(x,y, c=c)\n",
        "      plt.xlabel(\"Time\")\n",
        "      plt.ylabel(\"Pitch\")\n",
        "      plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (INPAINTING)"
      ],
      "metadata": {
        "id": "sKuGQU09BcRT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Pitches/Instruments Inpainting\n",
        "\n",
        "#@markdown Inpainting settings\n",
        "\n",
        "#@markdown Select desired instruments to inpaint.\n",
        "\n",
        "#@markdown Selected instruments MUST BE present in the composition for inpainting to work\n",
        "\n",
        "#@markdown You can stop the inpainting at any time to render partial results\n",
        "\n",
        "Piano = False #@param {type:\"boolean\"}\n",
        "Guitar = False #@param {type:\"boolean\"}\n",
        "Bass = False #@param {type:\"boolean\"}\n",
        "Violin = False #@param {type:\"boolean\"}\n",
        "Cello = False #@param {type:\"boolean\"}\n",
        "Harp = False #@param {type:\"boolean\"}\n",
        "Trumpet = False #@param {type:\"boolean\"}\n",
        "Sax = False #@param {type:\"boolean\"}\n",
        "Flute = False #@param {type:\"boolean\"}\n",
        "Choir = False #@param {type:\"boolean\"}\n",
        "Organ = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown Generation settings\n",
        "\n",
        "number_of_prime_tokens = 128 #@param {type:\"slider\", min:1, max:512, step:1}\n",
        "number_of_memory_tokens = 2044 # @param {type:\"slider\", min:8, max:2044, step:4}\n",
        "number_of_samples_per_inpainted_note = 1 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "temperature = 0.9 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
        "\n",
        "#@markdown Other settings\n",
        "render_MIDI_to_audio = False # @param {type:\"boolean\"}\n",
        "\n",
        "inpaint_instrument = []\n",
        "\n",
        "if Piano:\n",
        "  inpaint_instrument.append(0)\n",
        "\n",
        "if Guitar:\n",
        "  inpaint_instrument.append(1)\n",
        "\n",
        "if Bass:\n",
        "  inpaint_instrument.append(2)\n",
        "\n",
        "if Violin:\n",
        "  inpaint_instrument.append(3)\n",
        "\n",
        "if Cello:\n",
        "  inpaint_instrument.append(4)\n",
        "\n",
        "if Harp:\n",
        "  inpaint_instrument.append(5)\n",
        "\n",
        "if Trumpet:\n",
        "  inpaint_instrument.append(6)\n",
        "\n",
        "if Sax:\n",
        "  inpaint_instrument.append(7)\n",
        "\n",
        "if Flute:\n",
        "  inpaint_instrument.append(8)\n",
        "\n",
        "if Choir:\n",
        "  inpaint_instrument.append(10)\n",
        "\n",
        "if Organ:\n",
        "  inpaint_instrument.append(11)\n",
        "\n",
        "print('=' * 70)\n",
        "print('Allegro Music Transformer Inpainting Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "out2 = []\n",
        "\n",
        "for m in melody_chords_f[:number_of_prime_tokens]:\n",
        "  out2.append(m)\n",
        "\n",
        "for i in tqdm.tqdm(range(number_of_prime_tokens, len(melody_chords_f))):\n",
        "\n",
        "  try:\n",
        "\n",
        "    if ((melody_chords_f[i]-1280) // 128) in inpaint_instrument:\n",
        "\n",
        "      samples = []\n",
        "\n",
        "      for j in range(number_of_samples_per_inpainted_note):\n",
        "\n",
        "        inp = torch.LongTensor([out2[-number_of_memory_tokens:]]).cuda()\n",
        "\n",
        "        with ctx:\n",
        "          out1 = model.module.generate(inp,\n",
        "                                1,\n",
        "                                temperature=temperature,\n",
        "                                return_prime=True,\n",
        "                                verbose=False)\n",
        "\n",
        "          with torch.no_grad():\n",
        "            with ctx:\n",
        "              test_loss, test_acc = model(out1)\n",
        "\n",
        "        samples.append([out1.tolist()[0][-1], test_acc.tolist()])\n",
        "\n",
        "      accs = [y[1] for y in samples]\n",
        "      max_acc = max(accs)\n",
        "      max_acc_sample = samples[accs.index(max_acc)][0]\n",
        "\n",
        "      out2.extend([max_acc_sample])\n",
        "\n",
        "    else:\n",
        "      out2.append(melody_chords_f[i])\n",
        "\n",
        "  except KeyboardInterrupt:\n",
        "    print('Stopping inpainting...')\n",
        "    break\n",
        "\n",
        "  except Exception as e:\n",
        "    print('Error', e)\n",
        "    break\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "#==================================================\n",
        "\n",
        "print('Rendering results...')\n",
        "print('=' * 70)\n",
        "\n",
        "if len(out2) != 0:\n",
        "\n",
        "    song = out2\n",
        "    song_f = []\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 0\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    for ss in song:\n",
        "\n",
        "      if ss > 0 and ss < 256:\n",
        "\n",
        "          time += ss * 8\n",
        "\n",
        "      if ss >= 256 and ss < 1280:\n",
        "\n",
        "          dur = ((ss-256) // 8) * 32\n",
        "          vel = (((ss-256) % 8)+1) * 15\n",
        "\n",
        "      if ss >= 1280 and ss < 2816:\n",
        "          channel = (ss-1280) // 128\n",
        "          pitch = (ss-1280) % 128\n",
        "\n",
        "          song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                              output_signature = 'Allegro Music Transformer',\n",
        "                                                              output_file_name = '/content/Allegro-Music-Transformer-Composition',\n",
        "                                                              track_name='Project Los Angeles',\n",
        "                                                              list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 65, 73, 0, 53, 19, 0, 0, 0, 0]\n",
        "                                                              )\n",
        "\n",
        "\n",
        "\n",
        "    print('=' * 70)\n",
        "    print('Displaying resulting composition...')\n",
        "    print('=' * 70)\n",
        "\n",
        "    fname = '/content/Allegro-Music-Transformer-Composition'\n",
        "\n",
        "    x = []\n",
        "    y =[]\n",
        "    c = []\n",
        "\n",
        "    colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']\n",
        "\n",
        "    for s in song_f:\n",
        "      x.append(s[1] / 1000)\n",
        "      y.append(s[4])\n",
        "      c.append(colors[s[3]])\n",
        "\n",
        "    if render_MIDI_to_audio:\n",
        "      FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "      display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "    plt.figure(figsize=(14,5))\n",
        "    ax=plt.axes(title=fname)\n",
        "    ax.set_facecolor('black')\n",
        "\n",
        "    plt.scatter(x,y, c=c)\n",
        "    plt.xlabel(\"Time\")\n",
        "    plt.ylabel(\"Pitch\")\n",
        "    plt.show()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "qUeQjtsRBh0d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Congrats! You did it! :)"
      ],
      "metadata": {
        "id": "eoWDEy6CwDr6"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4",
      "gpuClass": "premium"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}