notebooks/Modeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "class CONFIG:\n",
    "    rootdir = Path.cwd().parent\n",
    "    specsdir = rootdir / 'structs' / 'specs'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making 1 named contant for each letter of alphabet bc time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import parse\n",
    "\n",
    "LETTERS = \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n",
    "\n",
    "def make_letter_validator(letter, msgcls=None):\n",
    "\n",
    "    @parse.with_pattern(f'[{letter}]')\n",
    "    def letter_check(text):\n",
    "        assert len(text) == 1\n",
    "        assert text == letter\n",
    "        return text \n",
    "    \n",
    "    return f\"msgtype_{letter}\", letter_check\n",
    "\n",
    "message_indicators = {}\n",
    "\n",
    "for letter in LETTERS:\n",
    "    key, check = make_letter_validator(letter)\n",
    "    message_indicators[key] = check\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A very lightweight parser that uses python formatting language. Pretty close to the smallest/simplest this can get while maintaining names/padding characters/etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "AddOrderShortParser = parse.Parser(\n",
    "            (\"{timestamp:>08}\"\n",
    "             \"{message_type:msgtype_A}\"\n",
    "             \"{order_id:12}\"\n",
    "             \"{side_indicator:1}\"\n",
    "             \"{shares:>06}\"\n",
    "             \"{stock_symbol:6}\"\n",
    "             \"{price:010}\"\n",
    "             \"{display:1}\"), message_indicators)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MESSAGE = \"28800011AAK27GA0000DTS000100SH    0000619200Y\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'timestamp': '28800011',\n",
       " 'message_type': 'A',\n",
       " 'order_id': 'AK27GA0000DT',\n",
       " 'side_indicator': 'S',\n",
       " 'shares': '000100',\n",
       " 'stock_symbol': 'SH    ',\n",
       " 'price': '0000619200',\n",
       " 'display': 'Y'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AddOrderShortParser.parse(MESSAGE, evaluate_result=True).named"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also generates regex patterns, might have some use browser-side"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'^ *(?P<timestamp>.{8,}?)(?P<message_type>[A]) *(?P<order_id>.{12,}?) *(?P<side_indicator>.{1,}?) *(?P<shares>.{6,}?) *(?P<stock_symbol>.{6,}?) *(?P<price>.{10,}?) *(?P<display>.{1,}?)$'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AddOrderShortParser._match_re.pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Result () {'timestamp': '28800011', 'message_type': 'A', 'order_id': 'AK27GA0000DT', 'side_indicator': 'S', 'shares': '000100', 'stock_symbol': 'SH    ', 'price': '0000619200', 'display': 'Y'}>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AddOrderShortParser.parse(MESSAGE, evaluate_result=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{timestamp:>08}{message_type:msgtype_A}{order_id:12}{side_indicator:1}{shares:>06}{stock_symbol:6}{price:010}{display:1}'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AddOrderShortParser._format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make [pydantic](https://pydantic-docs.helpmanual.io/) models out of them\n",
    "\n",
    "Up to here, none of the parsers have concerned themselves with validation or even the content itself.\n",
    "`MessageType` is the only slot whose content is checked, the rest could be gibberish so long as\n",
    "they have the right number of chars. It's assumed that those parsers will be used for internal,\n",
    "fast/compact data transfer that gets checked somewhere else up the stream.\n",
    "\n",
    "\n",
    "These are up the stream, used for data that should be content-checked. This is public sdk, available to third party devs.\n",
    "- Generated using the same public specs as parsers. These do content validation, not solely positional validation.\n",
    "- Available to devs who want OOP validation (vs \"#/ref...\")\n",
    "- The models/fields/types all have type annotations on them, so it'll work with static type checkers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from enum import Enum\n",
    "\n",
    "def oneof(name=None, attrs=None, **kwargs):\n",
    "    if attrs is None:\n",
    "        # use default name, attrs becomes first argument\n",
    "        name, attrs = 'enum', name\n",
    "    \n",
    "    if kwargs:\n",
    "        attrs = kwargs\n",
    "            \n",
    "    if isinstance(attrs, dict):\n",
    "        attrs = {v: k for k,v in attrs.items()}\n",
    "\n",
    "    elif isinstance(attrs, (set, list, tuple)):\n",
    "        attrs = {v: v for v in attrs}\n",
    "\n",
    "    return Enum(name, attrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def constant(val, **kwargs):\n",
    "    # Camelcase because these return constructed classes\n",
    "    return constr(regex=f\"^{val}$\", **kwargs)\n",
    "\n",
    "from boltons.strutils import iter_splitlines\n",
    "\n",
    "import re\n",
    "\n",
    "re_meta = re.compile(r\"^[^=\\w]*\\b(\\w+)\\s*=\\s*(.*)$\")\n",
    "\n",
    "def parse_flags_from_string(value):\n",
    "    for l in iter_splitlines(value):\n",
    "        m = re_meta.match(l)\n",
    "        if m:\n",
    "            yield m.groups()\n",
    "\n",
    "class MissingFlags(Exception):\n",
    "    pass\n",
    "            \n",
    "def parse_flags_enum(name, value, **kw):\n",
    "    vals = list(parse_flags_from_string(value))\n",
    "    \n",
    "    if len(vals) <= 1:\n",
    "        # Skip some cases (like with Y as true, empty space as false)\n",
    "        raise MissingFlags(\"Not enough flags found to work properly\")\n",
    "    \n",
    "    vals = {k: v for k,v in vals}\n",
    "    return oneof(f\"{to_class_name(name)}\", vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from boltons.strutils import slugify, under2camel, iter_splitlines\n",
    "from pydantic import constr, create_model, Schema, BaseModel\n",
    "from pydantic.utils import lenient_issubclass\n",
    "from enum import Enum\n",
    "from cboe.pitch import validators\n",
    "\n",
    "try:\n",
    "    from typing import _TypingBase as typing_base  # type: ignore\n",
    "except ImportError:\n",
    "    from typing import _Final as typing_base  # type: ignore\n",
    "\n",
    "\n",
    "def to_class_name(text: str) -> str:\n",
    "    return under2camel(slugify(text))\n",
    "\n",
    "\n",
    "PrintableAscii = constr(regex=validators.PRINTABLE_RE)\n",
    "Alpha = constr(regex=validators.ALPHA_RE)\n",
    "Numeric = constr(regex=validators.NUMERIC_RE)\n",
    "Base36Numeric = constr(regex=validators.BASE36_NUMERIC_RE)\n",
    "Price = constr(regex=validators.PRICE_RE)\n",
    "Timestamp = constr(regex=validators.TIMESTAMP_RE)\n",
    "\n",
    "pitch_type_map = {\n",
    "    'Timestamp': Timestamp,\n",
    "    'Base 36 Numeric': Base36Numeric,\n",
    "    'Alpha': Alpha,\n",
    "    'Price': Price,\n",
    "    'Printable ASCII': PrintableAscii,\n",
    "    'Numeric': Numeric\n",
    "}\n",
    "\n",
    "def is_msgtype(v):\n",
    "    vt = _get_type(v)\n",
    "    return lenient_issubclass(vt, MsgType) or isinstance(vt, MsgType) or (vt is MsgType)\n",
    "\n",
    "def _get_type(v):\n",
    "    if isinstance(v, typing_base) or isinstance(v, type):\n",
    "        return v\n",
    "    return type(v)\n",
    "    \n",
    "        \n",
    "def get_pitch_native_type(spec):\n",
    "    dtype = spec['Data Type']\n",
    "    name = spec['Field Name']\n",
    "    \n",
    "    if dtype not in pitch_type_map:\n",
    "        raise Exception(f\"Invalid data type: {dtype} - {spec}\")\n",
    "    \n",
    "    pytype = pitch_type_map.get(dtype)\n",
    "\n",
    "    if spec['Length'] == 1:\n",
    "        try:\n",
    "            # shitty hack to parse descriptions into enums\n",
    "            return parse_flags_enum(name, spec['Description'])\n",
    "        except MissingFlags as e:\n",
    "            # return as regular type, don't have enough to safely parse flags\n",
    "            return pytype\n",
    "\n",
    "    return pytype\n",
    "\n",
    "def get_slice(val: str, offset: int, length: int):\n",
    "    return val[offset:(offset+length)]\n",
    "\n",
    "\n",
    "def get_line_slice(line, field):\n",
    "    extra = field.schema.extra\n",
    "    parsed = get_slice(line, offset=extra['offset'], length=extra['length'])\n",
    "    return parsed\n",
    "\n",
    "def trim_starting_s(val):\n",
    "    #if len(val) == (model.total_width() + 1) and val.startswith('S'):\n",
    "    if val.startswith('S'):\n",
    "        val = val[1:]\n",
    "    return val\n",
    "\n",
    "def ensure_list(v):\n",
    "    return v if isinstance(v, list) else list(v)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cboe.pitch import utils\n",
    "\n",
    "class PitchMessage(BaseModel):\n",
    "\n",
    "    @property\n",
    "    def message_type(self):\n",
    "        # TODO: map all fields to snake case\n",
    "        return getattr(self, \"Message Type\")\n",
    "\n",
    "    @classmethod\n",
    "    def total_width(cls):\n",
    "        return sum([f.schema.extra[\"length\"] for f in cls.__fields__.values()])\n",
    "\n",
    "    @classmethod\n",
    "    def sorted_fields(cls):\n",
    "        data = [(k, v) for k, v in cls.__fields__.items()]\n",
    "        data.sort(key=lambda x: x[1].schema.extra[\"offset\"])\n",
    "        return data\n",
    "\n",
    "    @classmethod\n",
    "    def parse_line(cls, line: str):\n",
    "        # Assumes the line has all the proper data \n",
    "        data = {k: get_line_slice(line, v) for k, v in cls.sorted_fields()}\n",
    "        return cls.parse_obj(data)\n",
    "\n",
    "    @classmethod\n",
    "    def format_str(cls):\n",
    "        fmt_str = \"\"\n",
    "        for k, v in cls.sorted_fields():\n",
    "            if k == \"Message Type\":\n",
    "                fmt_str += \"{message_type:msgtype_%s}\" % v.schema.extra['const']\n",
    "                continue\n",
    "            fmt_str += utils.make_named_fmt_str(slugify(k), width=v.schema.extra['length'])\n",
    "        return fmt_str\n",
    "\n",
    "    @classmethod\n",
    "    def pattern(cls):\n",
    "        return cls.get_parser()._match_re.pattern\n",
    "\n",
    "    @classmethod\n",
    "    def get_parser(cls):\n",
    "        from parse import Parser\n",
    "        return Parser(cls.format_str(), message_indicators)\n",
    "\n",
    "    @classmethod\n",
    "    def parse(cls, val: str):\n",
    "        \"\"\"Parsed given string as PITCH Message instance.\n",
    "\n",
    "        TODO: Handle parse errors\n",
    "        \"\"\"\n",
    "        val = trim_starting_s(cls, val)\n",
    "        parser = cls.get_parser()\n",
    "        data = parser.parse(val)\n",
    "        if data:\n",
    "            return cls(**data.named)\n",
    "    \n",
    "    \n",
    "\n",
    "MESSAGE_TYPES = {}\n",
    "\n",
    "def make_pitch_model(data):\n",
    "    fields = {}\n",
    "    msgtype = None\n",
    "\n",
    "    \n",
    "    for spec in data['Fields']:\n",
    "        \n",
    "        kwargs = dict(description=spec['Description'],\n",
    "           length=spec['Length'], offset=spec['Offset'],\n",
    "           title=spec['Field Name'],\n",
    "           dtype=spec['Data Type'])\n",
    "        \n",
    "        name = spec['Field Name']\n",
    "        if name == 'Message Type':\n",
    "            msgtype = spec['Data Type']\n",
    "            kwargs['const'] = msgtype\n",
    "            pytype = constant(msgtype)\n",
    "        else:\n",
    "            pytype = get_pitch_native_type(spec)\n",
    "            \n",
    "        field = Schema(..., **kwargs)\n",
    "        fields[name] = (pytype, field)\n",
    "\n",
    "    config = {\n",
    "        'title': data['Name'],\n",
    "        'section': data['Section'],\n",
    "        'description': data['Description'],\n",
    "        'use_enum_values': True\n",
    "    }\n",
    "    config_cls = type('Config', tuple(), config)\n",
    "    subtype_cls = type('SubBase', (PitchMessage,), {'Config': config_cls})\n",
    "    clsname = to_class_name(data['Name']) + 'MessageModel'\n",
    "    model = create_model(clsname, __base__=subtype_cls,  **fields)\n",
    "    model.__doc__ = data['Description']\n",
    "    \n",
    "    MESSAGE_TYPES[msgtype] = model\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Same as last time, load the spec Yamls and use them to generate new code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TradeBreakMessageModel,\n",
       " OrderCancelMessageModel,\n",
       " AuctionSummaryMessageModel,\n",
       " TradingStatusMessageModel,\n",
       " SymbolClearMessageModel,\n",
       " AddOrderShortMessageModel,\n",
       " AuctionUpdateMessageModel,\n",
       " TradeShortMessageModel,\n",
       " RetailPriceImprovementMessageModel,\n",
       " TradeLongMessageModel,\n",
       " OrderExecutedMessageModel,\n",
       " AddOrderLongMessageModel]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "from cboe.pitch.specs import load_specs_in\n",
    "\n",
    "list(load_specs_in(CONFIG.specsdir, make_pitch_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AddOrderShortMessageModel Timestamp='28800011' Message Type='A' Order ID='AK27GA0000DT' Side Indicator='S' Shares='000100' Stock Symbol='SH    ' Price='0000619200' Display='Y'>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESSAGE_TYPES['A'].parse_line(MESSAGE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The models are all annotated, and can build json/openapi schema with expected data type validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Timestamp', <Field(Timestamp type=ConstrainedStrValue required)>),\n",
       " ('Message Type', <Field(Message Type type=ConstrainedStrValue required)>),\n",
       " ('Order ID', <Field(Order ID type=ConstrainedStrValue required)>),\n",
       " ('Canceled Shares',\n",
       "  <Field(Canceled Shares type=ConstrainedStrValue required)>)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESSAGE_TYPES['X'].sorted_fields()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'^ *(?P<timestamp>.{8,}?)(?P<message_type>[A]) *(?P<order_id>.{12,}?) *(?P<side_indicator>.{1,}?) *(?P<shares>.{6,}?) *(?P<stock_symbol>.{6,}?) *(?P<price>.{10,}?) *(?P<display>.{1,}?)$'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESSAGE_TYPES['A'].pattern()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Order Cancel',\n",
       " 'description': 'Order Cancel messages are sent when a visible order on the Cboe book is canceled in whole or in part.\\nNOTE:  Order Modification messages (4.4.x) refer to an Order ID previously sent with an Add Order message. Multiple Order Modification messages may modify a single order and the effects are cumulative. Order Modification messages always reduce the remaining shares in the referenced open order by the number of shares indicated. When the remaining shares for an order reach zero, the order is dead and should be removed from the book.',\n",
       " 'type': 'object',\n",
       " 'properties': {'Timestamp': {'title': 'Timestamp',\n",
       "   'description': 'Timestamp',\n",
       "   'length': 8,\n",
       "   'offset': 0,\n",
       "   'dtype': 'Timestamp',\n",
       "   'pattern': '^[0-9]{8}$',\n",
       "   'type': 'string'},\n",
       "  'Message Type': {'title': 'Message Type',\n",
       "   'description': 'Order Cancel message',\n",
       "   'length': 1,\n",
       "   'offset': 8,\n",
       "   'dtype': 'X',\n",
       "   'const': 'X',\n",
       "   'pattern': '^X$',\n",
       "   'type': 'string'},\n",
       "  'Order ID': {'title': 'Order ID',\n",
       "   'description': 'Order ID of a previously sent Add Order Message that has been reduced or cancelled',\n",
       "   'length': 12,\n",
       "   'offset': 9,\n",
       "   'dtype': 'Base 36 Numeric',\n",
       "   'pattern': '^[0-9A-Z]*$',\n",
       "   'type': 'string'},\n",
       "  'Canceled Shares': {'title': 'Canceled Shares',\n",
       "   'description': 'Number of shares canceled',\n",
       "   'length': 6,\n",
       "   'offset': 21,\n",
       "   'dtype': 'Numeric',\n",
       "   'pattern': '^[0-9]*$',\n",
       "   'type': 'string'}},\n",
       " 'required': ['Timestamp', 'Message Type', 'Order ID', 'Canceled Shares']}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESSAGE_TYPES['X'].schema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Test functions to parse the example data file into the models\n",
    "\n",
    "This can also obviously be achieved using the Kaitai classes. Doing it here to test  correctness, both sets of models come from the same specs, and the output should be interchangeable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from boltons.strutils import iter_splitlines\n",
    "from boltons.iterutils import bucketize\n",
    "\n",
    "#\n",
    "# Helpers to kickstart message/line parsing\n",
    "#\n",
    "\n",
    "def parse_line(val: str):\n",
    "    value = trim_starting_s(val)\n",
    "    msgtypecode = value[8]\n",
    "    try:\n",
    "        return MESSAGE_TYPES[msgtypecode].parse_line(value)\n",
    "    except KeyError as e:\n",
    "        raise KeyError(f\"No Message type found for: {msgtypecode}\") from e\n",
    "\n",
    "\n",
    "def parse_lines(lines):\n",
    "    for line in filter(str.strip, lines):  # Filter out empty lines\n",
    "        yield parse_line(line)\n",
    "        \n",
    "def parse_text(text):\n",
    "    message_lines = iter_splitlines(text)\n",
    "    yield from parse_lines(message_lines)\n",
    "\n",
    "def group_messages(messages):\n",
    "    return bucketize(messages, lambda x: x.message_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "RAW_DATA = Path(\"/tmp/data_med\").resolve().read_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "message_parser = parse_text(RAW_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "messages = list(message_parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<AddOrderShortMessageModel Timestamp='28800011' Message Type='A' Order ID='AK27GA0000DT' Side Indicator='S' Shares='000100' Stock Symbol='SH    ' Price='0000619200' Display='Y'>,\n",
       " <AddOrderShortMessageModel Timestamp='28800012' Message Type='A' Order ID='BK27GA00000K' Side Indicator='B' Shares='001000' Stock Symbol='SSO   ' Price='0000763800' Display='Y'>,\n",
       " <AddOrderShortMessageModel Timestamp='28800012' Message Type='A' Order ID='BK27GA00000L' Side Indicator='B' Shares='001000' Stock Symbol='SSO   ' Price='0000763600' Display='Y'>,\n",
       " <AddOrderShortMessageModel Timestamp='28800012' Message Type='A' Order ID='BK27GA00000M' Side Indicator='S' Shares='001000' Stock Symbol='SSO   ' Price='0000764800' Display='Y'>,\n",
       " <AddOrderShortMessageModel Timestamp='28800012' Message Type='A' Order ID='AK27GA0000DU' Side Indicator='S' Shares='001000' Stock Symbol='SDS   ' Price='0000549300' Display='Y'>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<OrderCancelMessageModel Timestamp='28800168' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800168' Message Type='X' Order ID='1K27GA00000V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800168' Message Type='X' Order ID='1K27GA00000W' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800174' Message Type='X' Order ID='5K27GA00000K' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800179' Message Type='X' Order ID='5K27GA00000J' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800180' Message Type='X' Order ID='1K27GA00000V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800180' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800180' Message Type='X' Order ID='1K27GA00000X' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800180' Message Type='X' Order ID='AK27GA0000DW' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800181' Message Type='X' Order ID='4K27GA00002X' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800179' Message Type='X' Order ID='6K27GA00001I' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800181' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800182' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800185' Message Type='X' Order ID='4K27GA000030' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800277' Message Type='X' Order ID='9K27G60000D0' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800277' Message Type='X' Order ID='9K27G60000D2' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800277' Message Type='X' Order ID='9K27G60000E3' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000E2' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000E1' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000E0' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000DZ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000E8' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800279' Message Type='X' Order ID='9K27G60000E7' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800282' Message Type='X' Order ID='9K27G60000E6' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800282' Message Type='X' Order ID='9K27G60000D3' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800283' Message Type='X' Order ID='9K27G60000DA' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28800283' Message Type='X' Order ID='4K27GA00002V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800285' Message Type='X' Order ID='3K27G6000006' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800297' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800300' Message Type='X' Order ID='6K27GA00001S' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800311' Message Type='X' Order ID='6K27GA00001T' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800319' Message Type='X' Order ID='1K27GA00000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800324' Message Type='X' Order ID='4K27GA00002Z' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800395' Message Type='X' Order ID='5K27GA00000M' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800455' Message Type='X' Order ID='9K27G60000D1' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800461' Message Type='X' Order ID='9K27G60000ED' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800499' Message Type='X' Order ID='9K27G60000E4' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800511' Message Type='X' Order ID='9K27G60000EF' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800531' Message Type='X' Order ID='9K27G60000EI' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28800531' Message Type='X' Order ID='9K27G60000EH' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28800562' Message Type='X' Order ID='6K27GA00001O' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800582' Message Type='X' Order ID='5K27GA00000N' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800728' Message Type='X' Order ID='AK27GA0000E1' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800728' Message Type='X' Order ID='6K27GA00001W' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800729' Message Type='X' Order ID='6K27GA00001V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28800758' Message Type='X' Order ID='5K27GA00000O' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28801018' Message Type='X' Order ID='AK27GA0000E0' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802100' Message Type='X' Order ID='1K27GA000011' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802465' Message Type='X' Order ID='1K27GA000011' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802520' Message Type='X' Order ID='4K27GA00003A' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802568' Message Type='X' Order ID='4K27GA00002T' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802568' Message Type='X' Order ID='4K27GA000037' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802575' Message Type='X' Order ID='4K27GA00002W' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28802661' Message Type='X' Order ID='AK27GA0000E5' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802666' Message Type='X' Order ID='AK27GA0000DV' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802667' Message Type='X' Order ID='BK27GA00000M' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802682' Message Type='X' Order ID='9K27G60000EE' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802688' Message Type='X' Order ID='9K27G60000F2' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28802688' Message Type='X' Order ID='9K27G60000EZ' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802694' Message Type='X' Order ID='9K27G60000FC' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802903' Message Type='X' Order ID='AK27GA0000ED' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802917' Message Type='X' Order ID='AK27GA0000EB' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802917' Message Type='X' Order ID='AK27GA0000EJ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802920' Message Type='X' Order ID='BK27GA00000K' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28802922' Message Type='X' Order ID='BK27GA00000L' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28803036' Message Type='X' Order ID='AK27GA0000DY' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803036' Message Type='X' Order ID='AK27GA0000EK' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28803036' Message Type='X' Order ID='BK27GA00000Z' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28803047' Message Type='X' Order ID='AK27GA0000DZ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28803224' Message Type='X' Order ID='3K27G6000008' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803224' Message Type='X' Order ID='3K27G600000A' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803225' Message Type='X' Order ID='4K27GA00003J' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803226' Message Type='X' Order ID='4K27GA00003K' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803234' Message Type='X' Order ID='4K27GA00003P' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803238' Message Type='X' Order ID='4K27GA00003G' Canceled Shares='000047'>,\n",
       " <OrderCancelMessageModel Timestamp='28803247' Message Type='X' Order ID='4K27GA00003C' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803247' Message Type='X' Order ID='3K27G6000007' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803247' Message Type='X' Order ID='4K27GA00002R' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28803685' Message Type='X' Order ID='AK27GA0000F7' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28804485' Message Type='X' Order ID='4K27GA00002S' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28804999' Message Type='X' Order ID='AK27GA0000EW' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28805614' Message Type='X' Order ID='AK27GA0000EW' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28805824' Message Type='X' Order ID='9K27G60000FL' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28805826' Message Type='X' Order ID='9K27G60000EU' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28806923' Message Type='X' Order ID='AK27GA0000E2' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28806922' Message Type='X' Order ID='6K27GA00001X' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28806923' Message Type='X' Order ID='6K27GA00001Z' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28806960' Message Type='X' Order ID='1K27GA000011' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28807197' Message Type='X' Order ID='5K27GA00000P' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28807197' Message Type='X' Order ID='5K27GA00000L' Canceled Shares='000080'>,\n",
       " <OrderCancelMessageModel Timestamp='28807220' Message Type='X' Order ID='5K27GA00000U' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28807224' Message Type='X' Order ID='5K27GA00000V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28807241' Message Type='X' Order ID='5K27GA00000W' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28808228' Message Type='X' Order ID='9K27G60000ES' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28808228' Message Type='X' Order ID='9K27G60000FF' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28808229' Message Type='X' Order ID='9K27G60000EN' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28809385' Message Type='X' Order ID='6K27GA00002F' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28809418' Message Type='X' Order ID='AK27GA0000FL' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28810393' Message Type='X' Order ID='7K27GA00000M' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811393' Message Type='X' Order ID='AK27GA0000FR' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811547' Message Type='X' Order ID='9K27G60000F9' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28811549' Message Type='X' Order ID='BK27GA000018' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811550' Message Type='X' Order ID='9K27G60000FN' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811554' Message Type='X' Order ID='9K27G60000EB' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811556' Message Type='X' Order ID='9K27G60000EL' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811560' Message Type='X' Order ID='9K27G60000GB' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28811763' Message Type='X' Order ID='AK27GA0000FP' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812228' Message Type='X' Order ID='9K27G60000FW' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812276' Message Type='X' Order ID='9K27G60000GD' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812376' Message Type='X' Order ID='3K27G600000E' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812389' Message Type='X' Order ID='7K27GA00000R' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812469' Message Type='X' Order ID='6K27GA00002K' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812535' Message Type='X' Order ID='9K27G60000GC' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812568' Message Type='X' Order ID='9K27G60000F9' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28812682' Message Type='X' Order ID='7K27GA00000T' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28812795' Message Type='X' Order ID='1K27GA00001Q' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000EP' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000EM' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000E6' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000FC' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000E8' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000FI' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='9K27G60000G8' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='4K27GA00003M' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='4K27GA00003Q' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='4K27GA00003L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='4K27GA00003X' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000EH' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000F5' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000F1' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813063' Message Type='X' Order ID='AK27GA0000EW' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000F4' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000F3' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000EU' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000FT' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000EG' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000EE' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000FG' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000E9' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813064' Message Type='X' Order ID='AK27GA0000FF' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813065' Message Type='X' Order ID='9K27G60000D4' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813065' Message Type='X' Order ID='9K27G60000E5' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813065' Message Type='X' Order ID='BK27GA000010' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813066' Message Type='X' Order ID='4K27GA00002U' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813066' Message Type='X' Order ID='AK27GA0000FY' Canceled Shares='000900'>,\n",
       " <OrderCancelMessageModel Timestamp='28813066' Message Type='X' Order ID='AK27GA0000FF' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813067' Message Type='X' Order ID='BK27GA000011' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813067' Message Type='X' Order ID='AK27GA0000F2' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813068' Message Type='X' Order ID='BK27GA00001E' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813069' Message Type='X' Order ID='AK27GA0000F2' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813069' Message Type='X' Order ID='AK27GA0000E7' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813070' Message Type='X' Order ID='AK27GA0000EF' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813070' Message Type='X' Order ID='9K27G60000GF' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813071' Message Type='X' Order ID='BK27GA00001F' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813074' Message Type='X' Order ID='9K27G60000G8' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813076' Message Type='X' Order ID='BK27GA000012' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813076' Message Type='X' Order ID='AK27GA0000F9' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813081' Message Type='X' Order ID='AK27GA0000FE' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813086' Message Type='X' Order ID='AK27GA0000FV' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813094' Message Type='X' Order ID='4K27GA000040' Canceled Shares='000047'>,\n",
       " <OrderCancelMessageModel Timestamp='28813096' Message Type='X' Order ID='9K27G60000F9' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813097' Message Type='X' Order ID='9K27G60000FY' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813097' Message Type='X' Order ID='6K27GA00002C' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='8K27G6000001' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813099' Message Type='X' Order ID='6K27GA00002P' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='8K27G6000004' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='8K27G6000005' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='BK27GA00000R' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='8K27G6000003' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813101' Message Type='X' Order ID='4K27GA00004E' Canceled Shares='000047'>,\n",
       " <OrderCancelMessageModel Timestamp='28813104' Message Type='X' Order ID='4K27GA000035' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813105' Message Type='X' Order ID='3K27G600000B' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813105' Message Type='X' Order ID='4K27GA00003E' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813105' Message Type='X' Order ID='4K27GA000036' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813105' Message Type='X' Order ID='6K27GA000026' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813108' Message Type='X' Order ID='AK27GA0000FS' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813112' Message Type='X' Order ID='9K27G60000F4' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813116' Message Type='X' Order ID='9K27G60000FB' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813116' Message Type='X' Order ID='9K27G60000GJ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813120' Message Type='X' Order ID='9K27G60000F4' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813127' Message Type='X' Order ID='9K27G60000GM' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813127' Message Type='X' Order ID='3K27G600000V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813134' Message Type='X' Order ID='9K27G60000GK' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813136' Message Type='X' Order ID='AK27GA0000FF' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813140' Message Type='X' Order ID='9K27G60000GP' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813142' Message Type='X' Order ID='4K27GA00003H' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813144' Message Type='X' Order ID='4K27GA00003R' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813149' Message Type='X' Order ID='9K27G60000F4' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='4K27GA00003L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='4K27GA00004B' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='AK27GA0000GD' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='AK27GA0000E7' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='AK27GA0000GB' Canceled Shares='000900'>,\n",
       " <OrderCancelMessageModel Timestamp='28813151' Message Type='X' Order ID='AK27GA0000FE' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813154' Message Type='X' Order ID='3K27G600000C' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813170' Message Type='X' Order ID='4K27GA00004L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813174' Message Type='X' Order ID='4K27GA00003L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813176' Message Type='X' Order ID='3K27G600000X' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813185' Message Type='X' Order ID='9K27G60000F4' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813187' Message Type='X' Order ID='AK27GA0000F2' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813210' Message Type='X' Order ID='AK27GA0000FQ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813215' Message Type='X' Order ID='AK27GA0000F8' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813217' Message Type='X' Order ID='AK27GA0000DX' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813224' Message Type='X' Order ID='AK27GA0000EY' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813224' Message Type='X' Order ID='AK27GA0000EO' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813224' Message Type='X' Order ID='AK27GA0000EA' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813225' Message Type='X' Order ID='AK27GA0000EI' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813246' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813247' Message Type='X' Order ID='6K27GA000028' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813249' Message Type='X' Order ID='BK27GA00001J' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813249' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813249' Message Type='X' Order ID='6K27GA000028' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813251' Message Type='X' Order ID='BK27GA00001K' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813251' Message Type='X' Order ID='BK27GA00001L' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813259' Message Type='X' Order ID='4K27GA00003L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813259' Message Type='X' Order ID='4K27GA00004M' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813264' Message Type='X' Order ID='4K27GA00003L' Canceled Shares='000200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813264' Message Type='X' Order ID='4K27GA00003D' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813264' Message Type='X' Order ID='4K27GA00004H' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813265' Message Type='X' Order ID='3K27G600000U' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813267' Message Type='X' Order ID='3K27G600000Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813268' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813271' Message Type='X' Order ID='BK27GA00001M' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813270' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813272' Message Type='X' Order ID='BK27GA00001N' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813272' Message Type='X' Order ID='BK27GA00001O' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813289' Message Type='X' Order ID='BK27GA00000N' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813289' Message Type='X' Order ID='BK27GA00000O' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813289' Message Type='X' Order ID='BK27GA00000S' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813289' Message Type='X' Order ID='BK27GA00000P' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813290' Message Type='X' Order ID='6K27GA00002T' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813295' Message Type='X' Order ID='6K27GA00002U' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813300' Message Type='X' Order ID='9K27G60000EJ' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813300' Message Type='X' Order ID='9K27G60000EK' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813300' Message Type='X' Order ID='9K27G60000EG' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813300' Message Type='X' Order ID='9K27G60000EM' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813301' Message Type='X' Order ID='9K27G60000GE' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813310' Message Type='X' Order ID='9K27G60000GS' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813323' Message Type='X' Order ID='AK27GA0000EC' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813325' Message Type='X' Order ID='BK27GA00001G' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813327' Message Type='X' Order ID='BK27GA00001H' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813432' Message Type='X' Order ID='AK27GA0000FU' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813433' Message Type='X' Order ID='BK27GA00001C' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813458' Message Type='X' Order ID='9K27G60000EX' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813458' Message Type='X' Order ID='9K27G60000FJ' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813458' Message Type='X' Order ID='9K27G60000F4' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28813481' Message Type='X' Order ID='1K27GA00001S' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813486' Message Type='X' Order ID='9K27G60000G1' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813486' Message Type='X' Order ID='9K27G60000EP' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813497' Message Type='X' Order ID='BK27GA00001D' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813535' Message Type='X' Order ID='1K27GA00001U' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813555' Message Type='X' Order ID='6K27GA00002Q' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28813573' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813576' Message Type='X' Order ID='BK27GA00001P' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813576' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813578' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813585' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813588' Message Type='X' Order ID='8K27G6000002' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813588' Message Type='X' Order ID='8K27G600000E' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813587' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813589' Message Type='X' Order ID='BK27GA00001T' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813589' Message Type='X' Order ID='BK27GA00001U' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813588' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813591' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813592' Message Type='X' Order ID='BK27GA00001V' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813592' Message Type='X' Order ID='BK27GA00001W' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813594' Message Type='X' Order ID='8K27G600000F' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813594' Message Type='X' Order ID='8K27G600000G' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813596' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813598' Message Type='X' Order ID='6K27GA000029' Canceled Shares='000500'>,\n",
       " <OrderCancelMessageModel Timestamp='28813600' Message Type='X' Order ID='BK27GA00001X' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813600' Message Type='X' Order ID='BK27GA00001Y' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813868' Message Type='X' Order ID='4K27GA00004R' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813868' Message Type='X' Order ID='4K27GA00004S' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28813869' Message Type='X' Order ID='3K27G6000012' Canceled Shares='000100'>,\n",
       " <OrderCancelMessageModel Timestamp='28814280' Message Type='X' Order ID='9K27G60000FI' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814280' Message Type='X' Order ID='9K27G60000EV' Canceled Shares='005000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814281' Message Type='X' Order ID='9K27G60000G8' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814282' Message Type='X' Order ID='9K27G60000EQ' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814383' Message Type='X' Order ID='7K27GA00000X' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814399' Message Type='X' Order ID='7K27GA00000W' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814405' Message Type='X' Order ID='1K27GA000020' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814474' Message Type='X' Order ID='9K27G60000F7' Canceled Shares='003200'>,\n",
       " <OrderCancelMessageModel Timestamp='28814484' Message Type='X' Order ID='9K27G60000FP' Canceled Shares='002000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814564' Message Type='X' Order ID='9K27G60000GQ' Canceled Shares='001000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814667' Message Type='X' Order ID='AK27GA0000GJ' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814678' Message Type='X' Order ID='AK27GA0000GL' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814691' Message Type='X' Order ID='AK27GA0000GR' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814691' Message Type='X' Order ID='AK27GA0000GQ' Canceled Shares='003000'>,\n",
       " <OrderCancelMessageModel Timestamp='28814692' Message Type='X' Order ID='1K27GA000024' Canceled Shares='002000'>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in messages if x.message_type == 'X']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}