CNN 알고리즘 구현하기-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "71a42d08",
   "metadata": {},
   "source": [
    "# CNN\n",
    "CNN(Convolutional Neural Network, 합성곱 신경망)은 이미지 처리, 비디오 분석, 이미지 분류, 이미지 인식 등 다양한 분야에서 널리 사용되는 딥러닝 알고리즘 중 하나입니다. CNN은 특히 이미지의 공간 정보를 효과적으로 처리할 수 있도록 설계되었습니다. 기본적인 CNN 구조는 여러 개의 층(layers)으로 구성되어 있으며, 주요 구성 요소는 다음과 같습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0e96c0c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hanjw\\anaconda3\\anaconda\\lib\\site-packages\\pandas\\core\\arrays\\masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "(train_x, train_y), (test_x, test_y) = keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5666a9c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAACqCAYAAABh9QLKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABl30lEQVR4nO39d5Ql2Z3fB37uDfu8yZfeVWW5rmpX7dFwPRiYwQDjiCFmdrhDoyFFypHSIfcsl3u0oiSKZpdnJYriWUo0ohGHIwLDcQBngAEGwADobrQ31eVtVvp8+byLeBFx7/7xslx3daMbyKrMyozPOX1On8p6LyOibtzvvb/7+31/QmutiYmJiYmJ2ULkdl9ATExMTMzuIxaXmJiYmJgtJxaXmJiYmJgtJxaXmJiYmJgtJxaXmJiYmJgtJxaXmJiYmJgtJxaXmJiYmJgtJxaXmJiYmJgtx3y/f/HT8ot38jruKb6hvvyBPxM/vxvEz+/H40d5fhA/w5uJx+CPx/t5fvHOJSYmJiZmy4nFJSYmJiZmy4nFJSYmJiZmy4nFJSYmJiZmy4nFJSYmJiZmy4nFJSYmJiZmy4nFJSYmJiZmy3nfdS4xuxghbv/ncR+5mJiYH5FYXPYg0nWRpSF0wqE3V8QrmngFSW8UZAjJZY3d0aQWPezFKrrnEZUroKLtvvSYmJh7hFhc9iAikSCcGsIvOKx81ETPdXlydp7/bvKrLEVp/psLv8DiRp7kK0lGTIlV6yHqDbQfi0tMTMz7Y+eLixAIw7j9zwwDozSETrqD0I4QYEi0baKFwCjXicobyEwaRkso2yTMOUSORGhAaWSosTa6CM+HSp2oVrurt3c3EY6DdBwYH6Z6NIVfFATTHodGKhxLr1AyDKDNscIqhlRc3udg+C7pZYvMvEvk+9t9C/ckwrKR6RTCsYkmS4RZB6vcRVxdRvf7KN+PQ5A3ISwbY3QYnXCICimCvIPejNzKQOMs1KBaR3s+qtPZ3ouNeVd2trhIA2EYyFQCzHdeqkgkqH50iuY+iZagLYhsTVCMwFQMfz/L8LclvSOjLHzKIiwFPHH4Mh/KX8ZTFr42Odkc5/UXD5JYk4w9l0N+f/eKizFcIhorsPFwhok/c5kPFS7zcHKeSaNB0QhwhcOoIfivRv6Ibsnk5PQE5z4+xv/x+lMcfS0P9cZ238I9icznCI5N0Ru2Wf7ZgGeOnOd7z97Pwd8wkfUOcnEF5XnbfZk7BlnMU/7UDJ1xgX6sya8e/h6WDDHQXOoN8+3feoyxFws4y004eykO1+5Qtkdc3n6ALOT13YkwJEgJhjH4f9NEZDNox3rH10SuTXdM0puI0KYGUyOciNFSE9uIqI2Mo3IpekMmYrrLwZEqvzb2fT6b9OmqPg3V57nEBK+NTOMHLmHKxL4b93+3kQZCCnQmiV9K0BsR/NTwST6dOsOwFCSlBTgAWMJg1jSItKZkzHPcWeSbo0fQCQekAVrt3FX2bXa5Ooq2/XqFbdHPWngFyf7JDX51+Dm+N3aQKGEhuxa82858ryEEwrQQqSTdUUFvMuKzsxf4a0NvYWJgCMnJ/kn+YPgRgoyB5e7Kt3XXcPfFRRoYhRzCtgciIiU66dKfyBImDLrDJv2sQJsQ2RC54O3zSWTeGZIxDMXR4bPsT1WwRIQlBisYhSBQBv/+sQyX0wX8yYA/d+wljrgrTJt1apHmUmhywp/je/XDmPMu6QVwKh47dNr8kRGWjTE1jsokWPpEke5TXeZGV/hQ4iLDUuCIwRDwdEhXRURAoEEBjoBhQzGZbrA2d4AUB2BtA9VoopXecStGc3QENTaENiWRO7gv+/I64dLytl6XTrq0Jg28kkD3XL7WeAhVt0HHYcabMWem6B0ZpTVpYTxd43OTl/jpwhsAKDRoRYQYhLQ1CK133fu6m7jr4iKkQCST6HRiICymJCgkqB1yCDKC9myEOexh2SEpt89Iqs1/PfNVHnPe3/d3dZ+3+g7lKEtzLsHruUnuL67w5wsvMmw4NJSirjRXghIvt/ZzqjZKclWQWg2RzR47a7r88RGWSVTK4pVcmg/2+Z+f+BJjZoMjlsIRg4eqUARa0dKCQEs8bRAhmDB8ctJmzG1yedjA7GVwOz1EpwthiFbbfHNvQ2fTdKfSRI4gSA1KuIZqGVja5utybPyCoJ9XiL7FifoERnvz3C/mOlExS+2QTWda858cfIFfyb6JKySRNkEorpflxc/tnuCuiYtwHGQ2C/kM5Q+P0hsVKBOUCVFCE4wEGG7ExFCDqUydhBGQMT1G7BZFwwPc9/V7NqKIX698mIvNEgu1PL26S7mRphEkSBl9Kn6KTmiz1krTqqQwKxbjCyHJ5R6itQsOBzdDYCKRQGYzqKEsa09k6I5rZmdWGDMbDEkf+bYAYEtprgQFVsMc364fpeYn+cLoK/xyZoUHUkt85enjNA/aFEcmyVwdwlxrEF26uqN2Lyrj0h0xCF1BPzf4s9wFl3ep4rlraMsgTGqitCKb9Ci5Hc7benBI/W41RnsQbUnCFIQJTVL6WEIg9/Dzka6LzOfAdQjGC0QJkzBl0E9LlCkIE6BvKoNPVBTp+S4iUMh+CEoRZVyilIXRCTDX6uD3UfXGXTnju2viIrNZov1jtGeTuH9qlf/b/m+Rl11GjDZSaFwRYaBxBBiAIQQSMBAk5fuLrSoU54Mh/uCVh0hdMbF8cHxA2pw27wMNTkNjepq8pxhuRRjdLub5RVSrTRiEd/IR3BWEZSJsGzFawpst0pqySf7cKn966nUeT17imBUhsbHErXH+snJ4uTvH680pXvv+YdyK4Nc/a/Enj/wWX0xf4GOf+Z+ZDwv8lbn/C/WzKYZOOKQXlndUerJfStA8AGFKQckHDd5ph8R2XpQQKNckyCusvMf+bJVj6RWeTRyOheVtqIRJP6fR2YCi2SYp3nnOupeQhTz9g+P4QxZrjxn0hyPy402Ojy4x6db5aPocedm9/vf/zsLnOf+tOcwu2A2NDKE9JfDGQ9w1l5FXEti1PtZ5DbtJXK4jBEW3y2FrnZIRMGpce/V/+EDydUBXDyazaPOQNiNNHGFthnYimsrFqhkk1jUyACO4sYcWCuxmhNkLkX6E7PYRvT6620PvhjRbITCKBXQ2TX8iS3PWpjsmeDJf5gF3gTGjgyVs5G1cfww0rgwwhcLwBWYHGr7LhuqTFII5y8YVFfLZLpVcgjCxwyZGIYgSkiCv0ImIhBsQhhL9PhcmdwRpDMKSrgHZgGK2i2OEdJWNiAQyiCAItz3hYLsRjoMwTbyMRZBXJDI+KTl4HwMifK2oR3ApKHLCm8JqSQwvRAQ7Z2GzlchkEmFbqJEC7WkHryDpjwVkhtscLa3xVPYSw2aLQ1aFpABLCAwER7OrnBifQXYlQVoiIvAmQlIjHTqk6A4baMPGXnx/UaAfl7smLtrzMMtN3IzNWjfN1bCAJSqMfoBEmbf6Fq94h+gqm1qQwpIRn8mc4AHbJ9AKT2sueGMUT2qGnlsF9bbMJqUhCNCRGoRzghAdRShvFwgLIBMJKp/cR+VBgZ7y+OSh15l2a3wmc4Jpwycp3/1hz5oBP5M5QVL6vJg+gqpJKvU0/7z2FIfdVT6XXMAVMJevUB9P4OfTCMPYUeHv1oTJh46fRiG4VB+i3trWPQtGOoXIZqjOOHzxoed5MnWJr1Yf5vvrB3CXDcz5dVSztWvG34+CME3kwX30h1OsP2byM0+9xJHkKvdZG4DN5UByLhjj/1x9kjNfP0SirJk+2cVarKDbHXbcwd+PiTBN9LE5OlNJNh4wGX1miQOpBk/nLzJtVRgxWowaPdaiBL/beghPWYxaDTJGj/sTi/z1j5eJkLQjl0hLRq0Gw2aT7+07wr9PPYJct5krFxFXrt7xe7l7O5cgQHd7GN2Qrm9TjdKMmQ0Utw6O262qYRDyWo0KnO5M0Ilsyl4aW4bcn1hkzlwiYJDlVAuTJNdDwktX7vw97TCEadKelCSP1PjI5GX+27E/oiBdQKK4kRFxu2eekTYZCdN2BWVptAGhZ3K6NYZEEySvYghB0e6SSXkETvou390PJ0zBRwoXaIRJFlv57b4ccJxBJmRW8InMaR51qnyVh9lop7DaoBpNVLf7w79nN2MYhPkE3VEbbzTkc/k3mTVrFKVEoaioFPP9EmfWRxn/gY+zUIdaE9XpoHfhrk+YJt6QS2vCpDcb8F/Mfpt91gazZkBG2psRG5uqirjQG6HeT1BxUwxZHR5ILPDz6QXczQzQm+fSlHyT1yenuGIOEaTvTsnF3du5hCG0O5gbLfw3R/nb3c+xb7TCh0uX6EY2890iAL829n0+mbjxwvk64EIgWI/S/O2zn6P9YgkZgOkNDrP+xvQBGPIZKrZ5cuQqz6/OMtS5989OPgjCcTCGS6ihLN3piJ+bPs+jqSs44lahvhCEnOmPshwUeKk5C8CfHXmWj7t9uiqgrhSX/IM4FYPkqiJybF7yDnBltsiv5F+iuNM9tAUYKAyxA1az0sB7ZJbycZv2UZ8xo0mkNWu9DJ16gqGuHuys9ygymUSOjRDlUix9PEn3sM/Dc4tMm3XyUmEJkwjN79cf5qsXHoDzKey1GtSa6G4XvRl12BUIMXgew0OoXIrVD9mYx+t8bGyRQ/Y6FoqX/SL1KMnXaw/wyuoU7XoS96KD4cNrLoMF4aEOf/nBP2barvCUs0rJuLFzr0Rprm4UiMouZvfu7JTvqrjoMMQoVxh9sURnIcH83BQLB/L0exbmkgMCvvaZBp9MvHD9c56OeN3fz7neGJ0XSsz904tov4/qdhGGgdg/TVBMsvHQMH/wdBpdcRhpN9hLr61MuIRTQ3jDLvmZOr829H3yMsQVt+Zvn+yP8fXqg5ypj7B6agSA/Z+s8HH3TTpaMR9muegN45YhvdTH6lkkVw02giHqx2yGZYBEI3fYccvNyB2S3ysMg8oxm7GfWuB4YZFRIyAAyp00Rs3E6mj0Llt1fxBEKoU/W6Q7auM8XeHvHv0Kk2adWVNcz2QMdMj3Vg/gvJgms6AQS2u7z55ps/BXpFP0p4fojdrknlrnXxz712SkoihtNlTIy939nGqN88JLRxj9AZQ2AhJvXUR1ugjbAtOk/PkDfLnwKEcLq8yNbFC6KQpeDdMEGwncdYnR7d+VcPZdP9DX/QCn6oNwiGyTtkhj+4LkqkYbglc3pnk2/yqjRps5y6KlNC+39vNWbRy7zuDwvd8f/CckZrODBaRWbXoXE9hNEJ29YaUhHAeZcGFilMr9KXojgqOFChkR4m5mIgU6YjEKqCubr1Ye5tkLB1AdC7cxsMx5obKP304scrI3xUvVWS6slRguR1h1D6E1sm/RbBp42kKKkGm3ymouw1u5IWQhD4aB6nS3LyVZCGQigXAcQhfyRodAGztCZJQJGcsjbfgYQuApTavrYNckVjccnAHuUYTr0B216YxKxtNtxswGedlHYhMQsRYpqpFLpZpmZEXhbgTofn+7L3vLMfJ5RCFHMJZj4+EEXgmeKKyTlwpPw4VQcb4/wVcXH2B9PUd6QZJY97Fq3vW5UOZz6EySfk5wKF1n1q3iigiFgadDAq1Y7BdxygaJskb2grtSz3fXxUX1ehgnLpEyTdK5DCqXQgQRotlBuzaXS+P8P6Iv8JHRS/zXI99nOUryH048SPKCzejZwY7luqWHjohW1xFlSXa1TO6VNIQRUXnjbt/WtmCUhojGi1QeHHiF/WTpDB9LnmPYMK/HW6uqz2/Un+R0e4zX/+gIB3+/Q5QQtCclQVJw6cUZ/vqlL5K4bDPyWsC+RoB18Qqq2cI0DCzTJDt8H9UojSsCfi7zBk+nzvPnD07hHRnH3sgiLy5sm4GgMC3E5Bgqn8IvRRy21nFFgGVsf8hEWTDkdMmZXSTQ1QbBcorx0xHJxe6uO4z+IKihLOXjgmjC48+OnL4lRb4a9flq6yFOd8ZJnkhQ+KML0POIduH5lDowSeXBDM05+JnP/YCPZc9xzF4jJ23O+i6/VXuMVzamUV8a5vDJNkalgt6oosOQyPORtoV3aJTWjE3zgT7/+fgfMW12KcrB1L4WKZbDDN9fP8DoSwHJK01YXrsr93b3U5G1RrVaAMheD9FIQBQRtjsY6RR2fZxKPc1aPkuEJtAmeBKzA0YveodXlA766IBB3naletdvZ1vY3ErrpEu/6OINCX5i6ByfS5+kKEFupnUHOqKlJFd6Q1xpFEmsC4zzixiFHHZ2GKEkblkifZvMgiZ1pozudImqdXRwY5Vo9TSRlkgkw4YmI9tk0j36+Tyy72JZ2+h/KgU66RBkbbSrSMoQVwTbu3MRAmEMdoYJo09S9jEQRAikJ7BbIbLbJ9qrO5fN2p+wEFIqtpmwajibNS0Kha/hijfEpdYQdl0TrZd318G9EINaNNvGG3Lpjgn6YwE/m3+dp12frhK0dcDVYILTjTFWyzn2LfSRFxZRvd6tBZDSoZ8z6Q0LEjmPWbPLsHEjHF6NXC72R9hop5io9hHVxl3LTtxWV2TVDxBaD8IDWqH1oDYl9A26oYWCQXjs8CpXMkO4VZfC88YgOWCPIkwTozQECZfaoyNsHBeomR4PuAsUJXS1pqH61JXNapTnlDfJH585jLlsMz4fojtdRBCSURptmeQyLsoxMWvdwYqo30eHwbv+/qSwsETEh8bn+fpHCySXk8wsF7bNMVk6Dq0DWVpTBvnRKnnJ9RqJ7bkgAyObRqRSBDnNw6kF9tllLCFRWmB2BXbVQ3S9Pblzka6LSCRoTSR46oHzfG7oTR52llBYdFVASyte8qb5yusP4161Gb+0+9K0jUyGzsfvozNmUHks4mPH3+K+1BpzVpOukvyLxgM8V5vj1UszFJ51GK8q3IsrRJuJDDcjHIfKAwbpD5V5ZvwCqZuSeAId8Y9WP82zrx8mfcnEWlkcZCj23/393kq213JfRbdWeCuFDDX0JV40WMlkpOLjwxcYcjucHToCQg4qm3fTSuaDICQ6k0LlkrSmJc59dY6U1tln1clIm0boU44SrEY5zvujvNGcxlqwyVwBd6036B3iedBs3vhKeN8xWEsYGAiOp6/y8pFpym4Blb47RVm3vyCTbknSmdAcyjZJCwtXhMhtqsARUiBcF51KECUV++wyY0b7evaTDEC2PNijtS3CthHJBF5O8gul1/jF9AYKi0hrPK2oRhaX/BGSF22KZyLc5db1gundgkgmqB8yaR2I+NjxM/yv09/cdMxIUFPeQFguzpJ+02HsDxfRjRZRq3X7RbVt0ZsK+Cv7n+M+ZwXrFnFRvLYySeklg9RqgKrU7mrq+47q56LDkMzVkMi2OG1O8K2JKYpGm1lng6Th8/roYeTcDKLTI1pd23s7GGkgs2maDw3TGTNoHwn4zORl5hIbGGjKkc8/qX6U59f3U+0k6VSSyLZB4QokNyKMtr9lL2pW9igmupTd3MDderuQBmFSEGZD8nYPgI62KXdSBE0Hw7+7uwOZydB5ZIbuiElqsrEpLIr5sM/FYALTA9EP9tbY3Uy1Fa5D/6F9VI841I8pJqxbM7/OBll+q/YYL67Pkl7UpBa7iEZ7my566zGGh4kOjNMac2keC5jbv8YT2XkUio2oz6kgx5X+JK+c30f2hE32SoRud9C+P3Ahh82miBKZGqQuB6M57LzPtF2haHSRSKqRz9e7c5zrjeFfyjI238eu9iC4OzuWa+wscen3Sf7gAqk3E5jeLP/24FM8kF3ml/IvMZEK+WdzH6Hx0BDJ9T5mo4nePLvZE0gDaVtQzLP6tKB0bJ0/N3mKv1h4EQBPw6UwzZdeeZyh5y2KlYjpy61bLTLKW5fGWTTazGUqzGeKaGv7+pEIQ9LPgVPqMZmoYwhBS7nUq2nssonZubs7BFHIsfxxEzHX4U/Nvc5By2Qt8nnJm+Cl9hxWW2/aDfX3zO5bmBayVESnkyw94/LYZ05xLL3CIbMNNzm/fbt9lK+8ehx32WLfa1X02UuEu6WWBVD7xlj4VBpvPOIvP/1NfiX7JpYQBFpwKUzyb8of5kxthNL3LEa+dgnd6xE127dkYgrDQNg2slSk9dAInRGDI2OXedDeICkElrBYiBz+pzOfpLWUZeIFhf3cyeulIHeTHSUuaD2oXwlD3Jri3Now/cjgs7k32S8UxWyH9mSayHEoVCYxWx3oB+gwQnseqt3etS+sMAxEKolKJ4jyIQfzGxx0B1klG6rPc94s57wxrLJFai3CqfrIjQZEETg2CHHLIf2PiyEUpogQQsN2+g4LgZaD3j7X+vkE2kR7BmZHIIM7tHO5toJMuINQj2NDMoE/WyQYDjhYqjFlVzEx6CjJm71pTjfHMLsawnBPFVAKQ6LTScJ8gn5O8XB2gTm7jLsZwmmoPh2ludgZxi6bOBWQXY9wC8frtrI5VsK0hTessEo9Zu0NSkaCjajHsjI46U/x6uoUzXKamY1o0DMpCK8LizDNQVv3kWFUMUNvOElj1sQf0kwl67hC4GlNI+xxvj9Ls5wmsWrg1Pxta6O9s8SFwe5FByHZNzfQosTyTIb/8KvH2Tf0ff7s7A/4zhePcKVR5NzjJcxOAbcssJua3CUf8/mTu8OA8jbIfG7gKr0vxYOHrvBXxr/BmOFjCIdvdef4W1/7AsllyfTLPu6pRbTfv566KTZrXvZKKKYSpnGXTdILGrN+B3r0SAOZSiIcm/DINL0Rh9aUQfOgQhR9/txDz/F06jyHrBqQ5LneHP/6+Y/grppMX+rcaLa2RxCJBK0jedpjBun9NT6XfoucjEhKB18HfKM7w+udGZ49cYi5P+pjV7qocmW7L3vLuFaHVZtx+PATp3k8d4UHnRUUNi/4Y/xB7UG+O3+QoX+XZOxqF3OhTOj51xM+hGlijI2iUwmWPz1C6+kehVyLz0y8yqRT4yeSZ0kLi+/6Gf595XFeXp1m9DsG+bNNjKUNwm1acO84cblWv0K5Su6UiQzzzHeLtAoGD7oLTI9WeCs7ze+Ih6i3k7RTKeyaxG5ZZG17029oB7fi/RERjk2Qd/BzgkPpdR62ARwirZn3S2QvSHKXAtxLZcLVW/PYd9eT+OF42sLsgd1W8EEzYzZXmbf9kRz8TFgmIplAJFy8YZv2uEF7RjNxZJ192SqfyrzFMcsjKQcpoWtBDnfZJLWkMasdoj0i8sDgedoWXk7iD8FMtsmUyXX3CE9HXPaHOdkYxy6buJeX0fUmqtfb5gvfQgwDbIsgLfho/jxPJC5ft1JaCgqcrY/irybJvlEmOn+J8NoYFBJhCITjoDNJolyC9rTm5+97g0OJNb6QOUdO2lxrorYeZjhVG6VeTnNw3kOeu0q0jYvtnScum+heD7FRI+WavPz8YX55bpyfmL7Arww9zwOJBazpiEaU4PLUEFU/xamRGYQ+hl0PSVwooxutQcHlLtnJBDMlFn/CIhjv83DqKhLJuaDPyf4Y31k/RPZKSPJiDX2HUoINoZAIFPoWQ7xtD4vdBktERDaErgDz3c+DjHwOkc9d76uiTYNgNEuQtYgcOfi8GHjYaSnojgi80sDUU1sabWhEoU8i2aFgBySsgZAF2sTTCqlDkkJSC5Mkypr0ajhIQd4jGKUh1PQYnckUG0+HTM9u8PnRtzAQlCOft/pDXOyP8M9f+QipMw4j5yN0vYnu9XaPbxiDsKAwTZQFY1aDUaOPK0wirfnK6kOUvz3B0KpGRGpQGD03gTeSwM9LuiOSyIHemEInIx4+fImfzJ1i2Gi9wzvwRHealbMjpFYkVq2F8v13pC7fTXasuCjPA8/D6AdMfP8QnfNZvvvTc/ynw99h2uzyjFu/3vAq0pr/Z+bDfLX3OG7ZYaJXHHSx02pblXsr6U64zH3oKo8Xr/KYuwDYnOmP8vXqg8wvlbjvYo3o7IU7eg2GkKAVCo2B3hEWK7fDFiHK0YSueM9kA5HLEkwWBx0hDYEyJdUjDr0xTZiAKBeCoRGGRhiKnzh0nr88+kdYQmEJRaQFXW3iaZMXugf51sYRlBZ42sLTYGmFEopqP0VyXeEud9G7sMr8XRkq0DiaoTUt+fnHXuQvDX2XvFRIbKrK5Nuto5xsjFN83mbsm8voVoeoXt9dUQchBjsXKVE2TJi160WOgY44d3WMw19rILv+4ByukKP6QJrmAehP9Hn68CUmE3U+mT3FrFljyNDk5LWeTPZ1h/NIa863RsifEaTWIkS1se0L6x0rLtfQ/T5uuQ/aZvlijr81/Hlmk1V+OvsmRaPLhBGRky73JVb4w31tOukk9bUEqYxFYt5G9gN0FG37g/6REAIjl0WkUvSKkvvTVWacCkkRoVAsBwXO1EcQdWvQdOouUldJlns5vJ4NahsPXrVGKFBKEmiDSGuGzSb9WZ+67aBlgezIY+/8nBTURy26owIk6M0dSm8iQhb7mFZIITkYM/1wIFC+MvlBb45Am1TDFF1ls9AtUO8nWGzkaK1mMDIBHy+cY86qXjdP7UUWVkch2z30XSpg206uNf/yJ3NU7xf0R/scSqyRkQpXSAwhCLSk2k9R8xKYPaDbg206eL7jKD0YpxG0lEtXtXA2bfEnxmqsfWQMGWhECNqExmGNnOgxO9TggczyoCun9FEIOkoh6W926B3UAnZVQICm3EuRLCvccn9H+LDteHFR3S7G6+dJ2zYH1qa5+tJhTuyXvPqpaR7KL/HFwks85sAXMuf40GOXeN2f5u8WPsvGapKRF4cY8gadJqNyZUf1e38/CMNAHZymM5OicVjzc8XXuM/aoGgMJtEXGvtZOzlCZl4iendWPCMtiTZ3LQrFlX6Jt1bHEWsO0uveFSO8d0OE0A8MfDUYzk84Ff7+07/Japjj9dYMV9uF237u44UlPpI5j7zJpj8l+lgi3AxtWTSVywutA5T7aa40h/gHiz+J37GxVmzMriAzr0mWQ8ZaAVPVGu3DBb6z7whPJC7jij5prWkGLu5KG3V1aUe89HcUITAKeXQmxfrjDn/tC7/DQ84Cc5ZHbvMMSiJpKYsLzRJr1SyTtYhoo7KrQmHX0RqiCB0EGD5c6Q8zbHSYNkKS0uJ/u+/XOTM3SoRA6UGYa9hskpEeKRGSkxGBhrUowWqU3owYKPLSY05Eg7oWpagoh5W1PEdfXkGVK0S97Q+/7nhxQeuBKWKng7mUIqM0kZVmvlxAojmWnGTUuEBKCu63TSwxz/7hCpc09IbTRKUssmEiqnX0PSYuAMox6ackUTJiSHbIyUFL04CIqp/Erkuslr4jmWDCcRC2TegIDDEQFl8HeDpipZ+n13Bx2vKu75puRkcK04Ne22LVy7IYBRho9lkbjJl1LBEx4dbf8TmJ5vHUJZ5yBllJntYEGjwt8bVBE0kncmhFCepBkno/SbmZpr+exGpLUssCs6PJXu5hrTYQXh/d6+GMpPHCwYoy0uDrED8ysfrhICV0tyMkJBNEhRR+QfOxxAUOWg5wq4uDpy2qnSRRy8Lwol2dyaijCBFFGL7mnDdG3uiScZZxhOagZXJ4s5jU14NdbVdH9LXG11BVJl1lcaY/Tj1Kkje6FI02BooIH1ADq6cwh+6Zg7PmbTKRfTs7X1xuQlVrGJ7PULuA6RWoZKf5u/dN83fGPZ6eu8w/mP4PjBrwn01/m+WxAv8q9yHOzZXIXMww+W+bROXydt/CB0NIlCWJbMDUJGWAJSTlKKShLM4ujTL1Woi74Q1avm7VrzVNhGkSPnWU6n0utYcUY0YDXwd8s1firDfOl088yuQfGLgbHpS3zzBUt1pMfKtB/0SCk48d4U8+9RfIJzwO5cqkTJ9W6NLbtBJ6O2v9LM9bHZa8PC8tz+B5FlHDxuhIrKYc2JMHYPY0MoSRjsLshsh+hNn0Bm7eteag5cDMBK0npmjsN/jF/AKzZkA5kpwIklQ6Scai3Tt53oy0LRqPjlI9amAfaeAK9Y7OpwCvdvcRvlBgdEHhLtS2ded7p9G+jw5Diie7fOXff5gvjz3FX/j4d/iV3CtkpCAnXdrK52zgUFdJfr/+EGcbo1zZKKIupzB8gdUBEUH30R6/dOwVDrjrHLIuIoEv1T/E8+v7SV41d5RI31visrmDEfUGubUNRDJJojZJayrBi+YM3pRm1Ejw08kWigbu/oCvZR/gBfMgU8lt9L/6cRCgDUBqLBQGkoayWI/S6KpN6nwF0e5uXeLC5gGksG1a0w61hxRDszVy0ifQcKo3yauNaewrDrnn51HNFlFn+w6plefBayexgFEeZymZZzGv6OyzyCfeOzRQ7g1aNS/XcnAiQ6oJyTWFW4twVxrok+ff9WV9+3QpEhbNfQadKcVBd42CTLAYBiwHBXr+Np9L3U0si9akQf++Ho+PruAKbms5tOgXyF+IyJ5vwcbudjPXYQhhiHV5jYlnJ2jOOLx+fIrPZd7EEgE5wNOKpbDAQlDk+dX9bCzmSV0xGX++h9EJkM0eaM2V5BhX9xdJGz79zed6sjHOwuIQherO6m56T4nLNXQUoTeN/1ILXcyuS/Nwgq4WKNT1VNk5e50P5S/x5sgE/ZkSNqA2qjtm2/ijEKHps9m8qy8Q7S660/nxi/I2c+uNbJrw6D68gk3lQcHhowsczq7T1SbnQ8Efrh5lfn6Y4iJozxv4Fe0Qd19nuc3QW3mCpKS3WGLl9huWd2B3ILMQYXYVdq2P0fGRtTbhB3imYcamO6ph2CdvdFBo3vQn+f3KQ3jlBCKo/2g3dY8gLBuZz0EhS29Uc2CszJH0Gpa4NU39XKA50x/j5fVpMpUQWWvdNQv47Ub3etirLXKh5s1vHeZPTO5H2hGGqYhCiWpbSF+SWJEMVTXJ9RBrtQVKoZMOKmHhDymezl9kn10mJSQdrah7CUTbxPTYUQkR96S4oKKBQHS7iEYTxzBIPfgYXWUS6AhHDHqPPGp73Ged5OT0JK8deYhMcpTkSXVPiwtAoA26ysHwBKpS3Rp7ByGRtoUoFlh9OkVnSvHUU2f4/05/ha6GK2GOM/4EC2+NMfK6IHexh2q2t9RS5sdFn7lA7oI5MNI0PoDfmVKDQ9fN9g9aq4GwfIAzOj9vIfd3ODBcYdhooYBnm4d44cwcqXkT/J3znO4EMuGiZkbwh5OofT1+aeJlDtjrt7j0KhTP9g7xOyvH2bhYpHR1nXBh+Z5LtPlRiRpNaLYxLhjMveHCtT5I11zer7UeiTbHYxQRBSFGOkX04BzesI010eFXs+dwhYnEoRX1qHcTOFWJ3Y7incuPijDNzSY7FiKVAtNA2xaYBv2cxhIKyY1JJdAKT2s6oY3hM3DI3UExyR+FYHP7fMkfwfDFYCD+GMIiHAdhGMhsBl3M4Y1n6I5r5JhHyWlTV5LzQYmvVB/hUmuIxJokUQkwWh5qh2X3bIc53/XfLcE0I2wjwkATaSh7aYyaidUaJB7sZkQqSXtfmu6IpJSvMWbVycseBoNoQkuFdDW83prh4uIw7rqB8Pp7RliA6+4jWkVEH2BRprVGS4EyBYahcIWJIywirYg0eD2bVB3MzqAn1k7hnhIXY7iEGingl5JUjzoEaeiNKFQm4pEjFxg2FJawr//986HFCW+aN1YnmD7TRl5eJmrd2xbeDaX5zfXHObU+RnJV/3ghKWlgTIwR5dNUj2YoPyKISn1+5fizPJm+xFu9Kf4/qz/F9y4dYPS3HZLrfWaWV6HZRnd7e2ti+IAoFGfWRxl6U5Ba7Q9CiLuYYG6Mxq+2+PDkZT5feINH7Y1BITMWDdXnG919XPZH+NazDzL3lT5WtYba2D3+YXcSIQTKlkSWQEpNpDWDSjdNS5sYl13Gnm9hbrQItzFz8+3sfHHZbOmLGDirBsUE3RGL9rQmzEUUJ+vM5mp8fOg8SXFj1zJYLbmsBHm8ro2s1Ykq1R0Vk3w/6LfFrPtaUu6l6TRcUj/OfLVp4a+ySfpDLp0xiXugwUyhxmdzb/Kg1eWt3hSnq6OIqwnyz18lXFza1Vk9W43vWYxUQqy6f9d7adxtgrTJJ6ZP8mtD32fUCDY9rzZ/Biz2hzjfGSG5KrFPzKN73t5Izb5DqE3HwEBLrLbAXKkNMkZ3yPkn7GBxkckkspBHp5O0jg3h5SXtGYE33cfJdnhscomS3eFgco0xs8E+a+O6HYxCEeiIcphlwSugeibixwwfbTubGpOXio+PXKDgdLlw4hCFdzFZfC+M4WH8h2bwCyblRyThjMfkyAqfHT+FKwO+3TrGVyKH33zpcYZeNJlcCVGN5g//4pg9izYEI3aLUSMgKW897ypHNv9+/mGqS3nGr6pBP5sgvLffx7uI6nm4lytY1RSr9cRmavf29VB6v+xYcRHJBGo4jz+SZO0pSTjh8+SBK/zp0ecYNlocscLNQ62bJ9fB/w+2jZpKlGall0N4ckcddH1gNoXFEJqMNPlM5gT3Jxb5fxUP/mjfV8xRftihN6r5+DMn+C/HvklehowaDheCkL8x/ye4UC4x8qxJ4d++hI4iVDwRxLwHWgoKZoeSMZj8bk4/rkQpapeKFM4I0le7A8fjeDy9b3TQJ7x0BeE4yPoj230575udIS5CDA7qTfN6xzpvPE3jgI1fEIj9bQ6VqjyQWWbMaJKXfSzsW4SlrXzKSlONXL7XPcqqn+ObC0doLWTJXDTQ97Ib7dvew7z0CYwW/ZEQ9eQxjJaHXK+hgwCCcFAR7DqIdAptW4SlDMo1iByDyBV0Rg1aRwMShR5TiRqeNlgIbS4FJie8g5xZHiVaS1BqDDJW4ongR8MwI4KUg9U2kR8ke+0ewpyeIpgaon7AZNhsIhGABHFjMRdgYLUFbk1jdIPblFTGvCfSwBgqItJJVOLeeXo7QlyEbQ/8iNJJao+N0pqRtOdCfuKRt5hJVPlE+jTTZpOMFCSFgcS+HgK7xnxo8I3OMd5oTvODZ4+SXBEUzwRMnVxGd73Becu9jIZICyxhMGEEFGWTjzxwnud++QjuepLRl9LYNR+j7SO9PsFojvZsAj8nqB6PcIZ6TBY3uD+3xoRT56Pps2SFT0UlWQ1zvNmb4fsbB1io5Uk+nyK9HJG+0LhtAVzMe2NstiXIpDy6I2lkYJPZjeIiBI2nJln6lKY4ucF99hqGcDfj/jcWfh3lkFwRZM82kWvVeBf8AZGpJMF9U/RGbBKlztuiNTuX7RGXzYI9IQfV4DKTRo0WCTMOnTFJb0yRHmvzTP4sE2aNY3aLgrzRa1uh6Oo+SmvqStHVBq94+3mpvo/zlWGSK4LMoiJxtUU4v7Att7hViNu8iJaQSCHYl6zw8lgXz0jSGbOIXInVtjG8iN6oQ2d80F8+O97iQHGDR/MLPJ68TF52mbMGO7lV32Q5KHCuM8Ll8hD9qstoReFuBHuq98idwDIUPQuUCcid1fPmx+WaRZCflaRHGxwoVEiJWzOVfB1QVyHLQQGjp5Edb7C7jvlACCEIkwZBSuJYOycb7Idx98Vl00Yey4ZijrCYojWdYOUZjVn0eGjqHMdzi+x3yjzhXiUpNOmb0osBypHPc94ki/0h/s2lJ6gt53BXTPLnFNm2Inm1hmz10LU70zhru7GEgdSaP5F7hYMPrTHvl/jukYM0PZcNzybomxRyVZ4sLZO3uhxPXWXYaDJitCkaAdXI4mudWTbCLH+4fpTFep7+uSzjz0ZYrRB7qY5od1HN1nbf6j1JdFPzNL27NAUYVOMbk2OobJLmfviFfac47K6Sk4JIK3wdEhDxze4o/2rlw5xZHmVyKYRac5DCHvPBsEz8gkmvJCj+EEujncRdFxdhGAjXBdchGErRG3Np7pN89NG3+Gj+Ah9JXOSwdU1Mbu8H1lKS071JzrZHaZ0uUjoLucse1vOnUL6P0nrXx3UNIXjAhgfsJVqpy/xU5gQdbVMOs9SjJIecVZ5yOm8LH0rAoaVCznnjXO0VuLg6TFR2KZ2B5B+9hepur33+rkNwvdPlbkEYEpVLERQT9IciPpI+x5jZwBXG9ZYMgVac9cZ568oE5oqDU2kPbIp2e8uBO4E0CF1BmISkde88vzsuLsKyBz3HpydoPjBEmBD0SpIwAf2CJhwKyA41eSZ/jkPOKkV569Tm64CzgaQSpXi2c5hXajMsNXI0L+WxWoLSGU16wcdab6HCXZbeqBVWzSO9YtIqWzzXm6OhljhoeWRuqiOwhGTY6JHTPhnp4WmLYdlFMmil2tUBnta80S/xVm+a56tznHh1P3ZDki6D3RxYx29nS9TdyEiqzenhYay2IGfuiOPNrUFKtG0SugY4ijGzwbD0kZudEasqpBpZfHfjINmXXZLrCqPcIArCH98Dby8S9ElsRCjLoNxJYwhxveW48fZsnx3EHR/xMuEikgkaDw6x+gWfYq7DM6NX2OdWOOYu8YBdwRWCpLA2H1rils+3VMj3ug9wrjvGH7zxAPlXbZIbivHX1geV4u0O2veJPqAX1L2AjiKMtSopPyA1Pcw3qsdYy+TIZV8jc9OZnitMpjYnr8EBfB8wNzv+RZQjQUO5/ObGE3zv4kHMCwmO/Poaemn1hqdWtLt7atxtDCGYS29wanqcXjuBsN6ni+a9gBCDzMOExEz47DP7pMUgyUahWI0crgQlzl4e59jvXCVaLxP2+7tr4XcX0f2A5GIbs5vgajMJ3Gg5vpPZUnERlj1IgXVsKOTAtggKCcK0RWPOYHq4xnS6xrHkMtN2hVmzxqhxq5hc6/1QjnwWwwRXgmn+YPUBFus5nCWb1HqEWwmg3kK3Wqh+sOtE5WZ0v4/oerg1xWuLU6wWsmQMjwfdBabNJlOmc0v2iBSDZ1iNfBYih2qU44XOgUHvkqv7kAvuwDZmBzUV2q1YIkJKhb43kns+MFqAEBrJQExhsLhR2iDQJoQC7fn3ZovxHYSOImTHw7IMQs+hofokRYQjBgsWZYB2bdhhIcctFRdjdJhwaojueIK1pyRBISQ72ma2sMrjmTV+If8KeemTkxGuELjinemZng7xteJ320f5J+c+SnMjxcgfW0xc9bA2KohKHe33Ua3W7q/B0HpQGd/uUPyBxK2N4BXH+AdP/jTGWJefP3yCvznyPM5tqvS/1Zvlf7n4CTZqGRKvJUiuayaW+7hLG4iuR1St3/372SMMUpF3YepxzLag+3300iqy6mKtHeF7vXEmzRpH7T6WgCCjCSZyWFJCuQJ6Zyy2t1RctGvTzzv0hiTs63BwuMZHSxd5KnWRMaPJYUtgCfv67iTSg7a50U1xw2oU0dImpzsTNFcyuKsmhTMt5Nl5lL/3VkHXGg2p9Q2SSuEUs3TGcnTDJKfHxlgbCnFEeH0quzasznljrK3mMcsWxTMhyYUWcq1KuLq2XbeyqxEaokgSKnlLtthuRejBf1oLFFxvdRFzB9Aa1e0i+n3MjmC+X8IQioN6DZAoR9PPWBjNgcO51jvD6mpLxSUczVE5ZtGdUHz2wFmezFzkgL3OmNElIwWWcAl0xIVQUY2SvNTbz2uNGUItUVrgRRYnL05ir1q4G4LpyyFW08dYLBP1vMFOZY+ifR9da2D4fcZ+YNLP2aye2cfPTP7fud1cZtdgcjHC6oQkLlUHYbAtbIUccytOLSC6lOasb7I6neUBdm8atw5DzGqHhNao1QS/1TrMPnuDp5wajthFiQs7DK00mXnNP3r1J9g/scGhg/+OjIwYP7rOghwhfybLWHdm0Jl2o7LtC/GtGwlC4JVs2vsjkpNtfnXoOR5zrv3wxrmKr0MuBSNc6Zf4/ZUHuHJxFBQIJZC+YOY7ivRLl9CeN2iuozXxMfPghY6aTWg2EatrOIAjBEPv+aHB6mXvSvLdw6z7pBdd2sKlEqZhF4sLUYSoNTGDEHcjxffrh6im0zxgV3B2/6Zt+9CKzIKP96rLpe4YlbkEE0aXL0y9zouZfbwoDlE8k8WqWohmaxeJi9a46z7Zcyn8So7/WPxpRjLv7J3ihyblRprAMzFWHbIrYmBDpEEGGneth/Y8dD/YEVu7HU38fHYMstUls5hC9g3++1d/hv+9VGf+yjCJeYvMgr7elns3oJVG+32EEGSvKJ575QjPpg/we6MP4pohlU4Sr2eTOWvBHgtj32msukdqxcYvmLzQPYBKXCZndHkos8SbExNU78uQ2LDIV3PoXm9bz6W39kD/tbNMnE8hTBMSLvo2fkoOmlxYH9xwP7jVDkJrdKdLFA/ImHsMdWWB1NoGadNk7A8zaNPgmL8yKBr0/cEufLegIqJ6HYSk8Ps9it9NgzFoLa2lIBV1QLfR7QWi1i7ewd1ttEZcWaJQayHDCb702GNcGBnhTxRf5WfTF3HvC/gXxoeoL2ZILQ5h1Bvg+dvWinxLxUV5HuzyjnsxMbdDhyH62kRaq23vxdwNNlv2RvUG1HenzdJORHs+iDZOPWSpnOEtc5xH0iWOWBVyRpfZQo0zXYcoYWLaFgQBepvs3OLTt5iYmJh7BNUPEJHCPbvK7Jcn8Ioj/L8/9jl+59BxDmfX+QuT3+MPkg/y+uRxnNUSolzbtgV/LC4xMTEx9woqQquIaHWNZKdLKpehMzbFOWeM7EGPp0ZWqWTSvJh5BJVyMBrb5wwRi0tMTEzMPYZWGt3rgRSU3gpwajYnzx7hw7N/FbNhMHvawyg30NvowhGLS0xMTMy9hopQXgSeh/PNJo5hIMSgPxZao/t9wm12MInFJSYmJuYe5pqLx04rTBBax8USMTExMTFbS2wGFBMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5cTiEhMTExOz5bzvTpSfll+8k9dxT/EN9eUP/Jn4+d0gfn4/Hj/K84P4Gd5MPAZ/PN7P84t3LjExMTExW04sLjExMTExW04sLjExMTExW04sLjExMTExW04sLjExMTExW04sLjExMTExW04sLjExMTExW877rnOJuQcR4t1/pvXdu46YmJidydvniC2cF2Jx2aWY+2dpPTRKkJI0ZyVhSmN2BYYHyXXF0HOr6EYT1e6gfX+7LzcmJuYuIx8+SvnxPNoELUBGUHq1iX7l5JZ8fywuuxRvf4nFTwrssQ7/7cNf5VPJRV7wh3i5M8eXLjxCcr2Ee1kig5AoFpeYmD1H9aE86V9eIe/0AOiENhtqiqFXxZbsYO5dcZEGQgoQEqRAJlxEOg2GBMMAQDeaRNXa4O/vkTCQsGyEIennTcxhj+mhOvusDQoywbRZp+WuUEjdR5BK46RchG1t9yXvKITjIMzN10JKhCERqRRYJlEpiz/kfuDvNPoKsx0g/ACxtD4Yk3tkPCIG76iRToFtQTFPVExhND3Uxfl413y3kQbmxBg6m6IzIXgoVyZh9Fnu5QiVRGi2bGzem+IiDWQqOXjxXRdMk2B2mPqhBJENQVqAgJFXi1jPd9GRQgf97b7qO48QyGIeMika+w3+3P0/4OHkPHOWBySYM2FYLvBcaYHvjY9h9FOkaknYqGz3le8MhMAoDaFzabSUYAjCpE1zLoGfkzQ/3OMvPvzHWCIiQqD0+8uHOdke57nLc6hKkv2/ncL6XhsdRaCiO3xD24wQCMNAplOoQzP0Cw6rT9rIRxv454Y4/I97hPML232VewojnaL8qRnqhyH30AZ/bvh7rIZ5/lHtEyxt5Cl199qZy+ahkzAthGUibGuwSzENdNJF2ya9EYfOuEDZEKQ1SE3vqoWdySB8nygM9sRqUZgm2raIbJh1Npg26zhCotA4wsIxLApWF+VA5NzY5e1JNlfVQorBbsWy0PkMQTGJNgTaEIRJg+6oxC9oHp5Z5K8Wz2AJg0grFO9vPL2YOosXWZxNjuANZXGyabTno7rd3T0mxeb4smz8IZdeyaA3GfL56Qv8fvt+tGMP/g128zPYKQiBdBxEJk1vRBBN9zhUKDNtdvG0RS+wCDwTGewlcZEGRjGPSCToHhujfsAmTEFvRKNchSj0SST75JNlHs7USBgBKdNHacl/yD1Id/QQ6eWIzDdPEzWb2303dxzd7SEAtzLEVzYeZj5T4k/mXmX/zv+XvusY+TyikCMqpqkdSdPPCJoHQI95IDRCKiyrz6GRMqNui88X3wD4QMICMGt2+dXR5zmbH+f/98lP0pq+j8K5kNSz59G9Hsr3d+UEKyxzsGuZGuHKLwgOHlri8ew6D6UX+F52bhAeKxbipJK7gDkzRe3pSbrDEvnhGn9m/xscTSwBsBrmWVsskFiwcKve1v3OLfumO4QwDEQmjcomqR2xaT3RI5fr8pnJ80w6dT6VPsUR652r70BHKARfs47RP+OSfS4Bu11ctEb3eqAVTlNzujyK0oJPZd7a7ivbkYhUknAkS2cywcZxQTQU8BPHzvLzQ68hURhCkRJ9jtktctJGbpaFXRMWhXpfv2fYcPipZINHnXVee3CG14amqIkM6deToBX0A9C7L0QmDANh2/SHE/z0Y2/yv0w8x9Wwy0KUppTuEKaGMdMphOfH4nKHiUpZNh4WhOM+/+nBF/mPcm8SaE0A1KMk1oZJckVjNv0PsGx6b3acuMhUCuE6UMzTn8wTpE0a+036WfAO+RyfXWQs0eLh1FXyRhelBWuRT6ChryW2UJQMAwuDw8lVTk+MMl8fR5UKGH5/sEraxecvOhpMVkZf0+vZ1PwknraA3Td5/VgIQTBdYuPhJL0RQepQjclcg0czV5kwaxibr5grIizEdWH5cXCF4PHcPFJofjB2FFXKIS0T6fkob2/8+6SkYFh3ydk9KgkD7ToIa8dNQ7sOZZsE+Yhsvsuw2cISknKkOR+UeLU5S2pZkL3ax6h1CLfod+6sf1VpIEtFVCFD7f4s609rjKLPr97/fZ5OnWfEaFMyAmwhBucIWnMpNDnVH6KpXKphmrzR5aOJBYYNk0+nTnNo/yp/J/ocvdkhEoBcWiOq7WJxCfrooI/VjggaDqtuhq5ygO52X9rOQkhq9yWJPlvnaHGDvzb1dWbNLikhscQNIZEYGO9VjPoByEibP5M7wa9k3+TzR0Zp78+TWLcxa3Xwti4csZMZkgkKUjOTqrGcO4BbSGLUne2+rF1PlDTJTzR5dHSROXsdV5jMh2m+Un2E5xb2M/1aF+OVM0ThVknLThCXzYMmmc+BY9PfV6JXsmlPSZzRFhOFBo+nLnHcGRxMu2IwED0d4mnF694MJ7pT9CKLVuAy4rZ40Flm2ICMjBgzmuQcj6Yr0a6JMHa5483mIbWWAqTGlAr5PsM3ew4BhlS4Rsiw0WPYuDHJRVqjUPg6pBxpPD0IvV7b0UTcKjiuiEgKjS0EaWG9qyBlpA1Awg5QJmhzM51+j2AIiQFYIkJLgZYCsUXiveuQBtIdpMaLbGaQwNRqE9Uag3Dq+zink66LcB26WZNcwqPktAHoqoAr/RKna6N4lQRGu4Xa4gXOtoqLcByk46D3TbD4k0W8kkYcbvPA+AJPJat8OHOBvNHhiNUgKczroYmG6nOqn+Fcf4y/962fpfja5ssp4LUJwaFfXuNg7hJFaZIUAeOJJhtpSZixcazdXdchEwmEbeEVDYrjVY4UywwZHWAPZ4XdDq1IliOWzhV4dcamOu4ya94Q4a4OWI4MFsIi/3jxEyw1cu/5dePZJo8WFhi363wudZoJM16Nvx1jD4noVmAMDxHtH8MvOqw+ZdEfihh9TlD4xkW056Ha7fcWGCHg8D5aB3JsPGjwS6PneSx5mZZyecF3+OeXPkL4tRIT6wq5Vt3yJej2iothgOMQFhK0DkS44x3+1OGX+aXcKySFvmkleeuL2lGaq0GRs90xMucNRp4tow0DTIl1NMuSXwDAEgaWMAbZYxYoS4LcxQNcbKbUOg6hIxhOdZhM1HFFxM3iIjdX33qPLxjNboRTMenmXJrKJdCt6z9rKU05SnGlP8y51WHCtSSDCrObuOkBtoddbBnRTLl8LHn+PX9vpDVKCwwNe3VTKd/+LGPegUi4+EMO7TETfV+bB8bWuXx1jqLrDGqkhHzvRBAhCQsJ2uMGfinigLPGpFnnSlCiHGbYKGfYf66PXemhOlsfNt9ecZkYxdtXpHbY4cCxBZ4ozvN06jx5CdZtDlDLkc9GZPHV1uP88xc+hlU2mTzdh2oDmUygssltuIsdhmEgLIvIhclkgxmnSlLcOgBzZpfesEb2DZJjOczmENrvo1qtd/nSXYjWOAt1hu0i3RWT/6T3a+jMTfHmQCJ8idGV5C6AU99UgWshHK3hprwaP5vgwvABTozsZ/LTddzs6xTljTAYDEK5z3t5zvtjrF4tcnixh1Fp79lMKWWCtuR7G6zuYfz9JRY+aaBLPj9z4DTHksv8/bF9hOMFjLqDaHfQ/ruLi5CCxj6XxlMesxMV7nNWsITiX618mFOL46RPOrgLZUSzQ9Tf+nPo7RMXIQhHc1SPOjQPRfwPs9/gGbeORGII+7YfWYtsTvoTfHXxAWZ/G5KXN2B1g6hWwygNQSwuA4sN2yJMCPYnN5hz1nDf9u4WjTbBcEhXmXhLDulaEVlv/fBt9i5DXZoncXWJZCJB6fkh9E1ZS0IpCCNEEKI2qj90ZZdNuIh0Cv+Bab718BEeSVzBsGpkbloj+Vrxjcb9vLQxS/KKhXHxCrrVGtS57AEira6HxgwU2gRl7vFC3vegOePw+Y+/zOPpy3wscYWiYfA/jvfojSdwDYmxZL73wsQwaO0T/IVHnuWgu8oxK2I5ijhxbpqhF03y5z3Uhfk7lj27rTuXIGPhDWtEoU9GeljixiALdERLhQRAObLpaJv/0DjO8+v7Wb88xMGah2h1r7+YwrIIUzahK3Dk1mU83HP0AzD7WG3N640pAA5ZFUo3/ZURs0VpvEHFTtNZcLDrGWyAtfKurLd4N3QUoZVGSolsdxHmTZNcpCCK0GGE9vwfatUibBvyWYKMQc7ycWXwjr13pDXNMEHLc5ABEIaD1PE9RKQH9+vIkCAp6GdN3DgV+V2RQmMIjSHAQCCkvu4e8a4IgUwmEekUYVIzajUYMtqbZ9bRYMOtQWg9SAy4Q2zjzkXSnjBxHqpzfHiNYdkFbhy2V1XIqf4Q62GGb9ePstjJM//iFOPPRRys+phnrhK1O+gwAECnk3QnXLySIGPsjbTOd6A1UaOJaHfIXZng1dcO8NbYOA89cpX95o2Q1+NOlf/1/n/DxWCYvx59kTDpUjhr4F4y33ObvevQGnSE6vXQ/f47s7Y2Xzz9ftIzh4vUj5dozUo+mllj0miTkbdOAAGw5mVoNJLk2xrt+7u65uq9KFktuuMaoQ3Sl+KIw+0QCjqhQz1KEmmI0EipUaZAm+9+fiwdBzE7SVhIEo70ecydJycDDOEQaQGGRpmgDHlHkyy2dcmgBdhmhETTUA7VqIenIUCwEOY44U2x3s9ysjJGvZkktSRIXaghWl2iVuvWl940CB1BZA/SHPcsKkKrCLMV4JZtenaClkoAN8QlI20esKEol0hkPYKUTZiQCCG2rDr3nkLr9ycgt0GYJhgGKpvAzwv6WU3O7OEIbqmXAYg0NH0X3TExPUDtnV1LpN+euh0QJTRhQqBv47Cxl7nmzB3ZIIXCQBEgCLRCqYEXm3iv8LVlEeYS+AUbK+GRkQGuGJz5dbUJoUCGINRu3bloRenNNhtiiDdGSvyFxyfIJXusrOehbmF2JE5FYPiQLCvGeork1QasbqD6/YGr7M1fZ5sEaUGY0jgy2J572kFYl1eZ+qMxGgeTvPnUNL+UXt/uS9p1CMtGHDuAP5qiesSm8UifQqnFA4kFMnLgEnEzDWWxeGKM6e8pkvNNVH+Xj1Ol0EohA0UrcFmPumSkSULYZIwe5Pv0ew6Ra8b91jcRjkP44ftp7HeoPBHx04UTjBkNFsIsp1SCYD1Ber6LUe+i3n4ILw2kbSGHh1j+UIrOpOLJ6QVyUlBV8L3uGK93ZsmctRh+uY6sNAmjO7cQ30Zx0cgLi4xWcvizRZbsHGupLMVzkF4Ksese1kodPJ+oWhuEEHh3ExNtGYSuQNkKW+zhM5dNwtU1xOoahd79LPXy2305uxJhSLzxNI39Jq39iocPLHAgXWafWcMV73y1OtoiPS9Jf/88uuehd7nlvtYaEUUIpemGNi0tsHREQkBK+jjJAD9lDTLGYoDBTrix36FyXDE7t86jziquEHynV+SyP4JVl5jrDXSnhw5uneeuebmpbJLWXERprsqT+ctkpM1yFPFmd4ZXa9NkFiL06YuEQXhHE3i2Nyzm+9DpYq/ZFM7YhI4gvdTH2eghuj663YWgD+9DXaOEhT+kifIhGblHz1zeBSkUcrOiXKG3xCdrL2KMjkCpQJhP0J526acFjUMQjfYZH6vxdPES41aNjIy4+dXqqoCFSHLC24fh60GywF7IyositN/H6IZcbeR5rref484Chdsng+5tNntUyVyWzoRgaK7GQ8UlJLAcmvzvix/l0voQ2asMhMXz3hHSMqbG6RwboTVpMrS/wsfGLjJh1VgOfb7efphff+FDOGsmM4vdwZx6B0NisM3iorpd6PUQlSq5K4sgBDoMB1k8cCP09T5exH7Owp/uMzo6yIyIucEg20Rez9S58ecghN6b5ywfFCEI58ap3p+kPQMHP36Fo9lVPpS+yCF7nZQIKUqJFOK6RdE1ykrzx50jvNDYj9XR18f4bkeHIbrdxqh3qC2X+J3cI8hRxf322nZf2o5D2haymCcazuHf1+Pv3/d7DBstbCE4F4xw+bkZhk5q8qcbRJXqbe1fukdGmP+8IDna5L8//DU+llhhNTI4HxT40pXHOPJPe8iLSwPz3i30EHs3tj8HcPMw9V1vVhoY+eygfgPe1YfIz0nspEfG8bE2w2JdFdDRimo/helpjF70vnZBMXsEaQxCCZaJyKQRUoLr3FLvch0hqO1L0J4W+OMBx/OLHE0sc5+9xoShsYR5PZX+mi/ZchSxGqU448/y7cphLlRL5Dp6MAbVHpF0rQcH0JHAi0wCvf1Tzk5CmCbCNJGFPP7+YXrDFsV8jTGjSYTgfJDgdG8SpyZIlPuIdu/WtHghMHJZhOvSKBkkRtrMFmtMmjXSwqIcJXitt49qPcVItUZYq921e9vx/9JGLkv744foDhs3jP420eKGI0fjkOL4+AoH0hsUpUekTV7tZ3i5O8eLV2eYO9tBXlndEw3D3k6kxW0bXEUa9B72gDGyaUQ+RzSco/xwmn5W0J1URMV3HrQLAYdnFvjl0ZOMmQ0edJbJyIiMkDjCvG5UGWlNVfVpKcnfXPw5Xjo1h1U1yZ2FXEORfWON0PPveEgi5t5ADhXRw0UaR3KsfqHP7OgKf2nqOebMkK90Zvhn8x9jcaXIvjd83DeuojudWz/vOPQ+dJjGPpPaYyF/84E/ZJ+1wZzlESD556sf5/k3DpE9b6I7S3f13nauuGz23xbJBJ1Rg86kQFkabep3eGIJDeZol5lUjXG7gS0GL+5qmOdMZ4yg7mJU1gkr1d3ft/w2KC1vEZb32+RqVyMEwnVR2SReyaU9LegXIkoHqjxcWn7HX5dC8dOFE3w+2dj8E/u2z3HgpAwtbXF2Y4T0BQu3rCm91kA2u6hyZU+OwdshhAYBei/av1xrsZ1MEBQGc9wzB87zU8UTHHeWSUuXRpRieSOPsW5jV5o3wmE3Y1l0h03aMzA8UeeZxCWGDRMw8HTE5WaR1LxJcm2zKd1dZGeIixAI00IYElnIo1MJ+tMFaocdvKKAxxscLlUwRYRtDF5MectkKTiYKvN0+jwp0cfTBhdCxT++/Azll0YZugK61YlXizGDMOvwECKZoPr0OOXHIMqHHNk/z3iyybH0MnN2+R0fM4TiiLXO201U305LhXyvN8e53hid83lmX/Gxmn3kWhXt+YNq/xgsIhJOn56TuCUasevZrJ7n0CxBwWXlEZf2cY/R4TV+duh1jljrFDfzbSasGvvHNrhqFlh7Okd28jFSlxuoc5dBCmTCReRztGYE5uEmjwwv4opBAfrvt49w0Rth48QIMy/72JXeoEvtXWSHiItE2NYgjW60SH8oQfm4g/OJDe7PV/irE3/IQ/aN1Z4hxE0tZ28IhkTS1X3OBibLYYGVUyMc+t0WstFF1Rt7I0Mn5j0RlgmlAv1CgvUn4a//1O8xY1V41KmSFoNzvXfrxSJ/iLAAtLTg+eYBTtfGyJ8D64/fQEcRYTz2bsGVARmnTz0RooydMQ3dca5HY5LUj2TpTEiSn1rnWw/+K5LC2gyvJq+HsKetCk8U5xlyO7xw/ACdKZMxM0fqijU4J8xliQoZetMhX9h/ikfT87hCshYa/M7qcS6Xhxh6U2P98RuoKLrr89+2GlfKRAKRcKGQo3u4RJCSdEcM+lnozoY8VFxnJlFFCkVXB5QjQVW5ZGSfWTPEvanHy80YaCQKlVR4IwkcKZDL1sAqJn7JY2CQmSjAFX1cEWAhbvG2ezd+WEp3UmgOJtfpK5OXiuMYY6PoXm/Q4CkOh10nK3z2Zyt0A4vILW735dwVZDqNLBUJh7PUD0p60yGPF1dvEpZBRmdTeXS0QukE+9wNLBlxZapINZOi0k4h+8dQlqCflvQzgsJElUOJNYaMNh2tuBKOcO7qGPZVm0Q5GGQmbsO8ty3iIkwThESODhNMFKjelyD5S6s8UVrgUGKNaauCKwLyxsCJVmnJqSDFt1vHeKk2y1x6g18b+j6jhk9Gmjji1gZgBhpXBGTHWqw/UiCzYFK6mkb3g1hgYm5wU08R+T7i/hJxw4vpNgkSAMOGw69m36Cafoufvf8QzSencNd9zFfPDVLvYwCYNSP+0sh3eCkzx78ufI694C4mxoapPjFCe0Ly6OdP8RfHvsOk0cYQzmAsaUVIxPnQ4kpQYsxs8LnUOUjBL+Zeoassvnr0ON995iBJq88DmQp5s8snsyc5ZjXo6oFz/Hea9zH2BxaF5xbQjSbRNs13d19chEDYNsK2iIppuuMO3THBZ0cv8KnMSYaNDhkZ0VIGFZWgqxyuBkM0ogRvNCa5XCkSKkmlkCQjG7haYYlbz1Kk0NgiopTucKWUw+pIdDqJ9DxUhz1nFnitiPLaZHjzitswFIEFei/VVSqFCBVGT3LRHyXQJsPGZTKyh6cFwbs8DMnAoTbSArW5g8nICAtwhSQtHSSSkpEgKfs4iYB+ysFyDcw9bCsvlIZQ4IUWnh4sBB1hMmr0mLBqqF0eFbuWbqzyKbqjEm9E83juCk85AeAQaU1X+1SVoqNNTvkzXPaHUa7kkNnGFZKiKTGEopk+gyEUOaPHEXeZjPSYM9sUDQcinwqCUBnIUKP7fbTWII333RZ5K7l7/6yb8UaZTKIOz+DnHZY/bjP7kQWOZ8v8XO5VMiLgH288w4vrs1TqadSqi+EJkisCq6ORAWRDzfyhHL//8w/z4cwFjjvLJG+aCywMJoyAomzyn818hzeHp/ntyw9RqYyRWimSOL9OtLw22CruoTCFIeQ7VtuugENDZV7f79Jdd8nugQlQByGsrGPVXKa+PcNvrz2DX9TII20STp/aWhazZoIauNK+g2sbHA3aBGuuxVypwjOlc/zF/FskN3sRqXh3fIN+QHJZMu+McLIwCbkr231Fdw9pIO47iDeVYf0Ri0OfvciR7BofS57D0xEtFdFSkm92jvI/vfJJqNlYbYHRE3gHff70Iz9gv1PmmcQlJkyH++waeeN1XBGRlyEWkJGDaTwpDabx+bnCq7z4azOc/twMpWctRr61hO55RHc5U/EuistmUyDHwRtJ0Bk1sR5s8KUjX77uw1SNFK9uTLNxqkRiXVI4F2I3QuwT80QbFWQmg0wlMYJ9nP3JUabdKofsWw0ZDSHICJsM8DOpCj+TqiDR/LuZn0CZNs5qanCoC7ve2+kaxs3hn5t2MJYQ7E9VWC7laGXd3d0C+hoqIqo3gAaJl0Om5ot4k1lW/QztpGb4LGQW+ohII/vvPT4i12Tlw1lO7ndxjYA/m3sT97aKtLfRQYBT0wRpg9VeZk+lwgvDoD+aorHPpHuwz9+e/R32mwYBEYFWdPXgHPnV1gy5F13SSxFWJ8TwI1b9BC/M7KOWTfKoexWAUcNh1ACJxc0tShQKV5i4hskTToN/ePT/ZOlQgb9e+1OUXk0jpURU63d1zrvz4rIZyzYnxvCOjOEVTdYfl0RjPp+buoiBoKH6nOhnOe/vY+lyiaHTgkQtIrnYRfaCwflMJkPw2CEqx1xac4rPFi9zzFkiI0IiLXmtb/Lt9jEApuwqSelzv73KQcvk4eRVfuuJGhu1JN5wkeRKgeR6QOJyDRGE8DYDN+15u+oA9ma785t3LpaQPJqex5SKL4+VEKMlDNtCNZp3xR5iu9F+H1lv4UpJ8ZRJZAvSyz5WuTMYD+F7H4SatkXhnIXdtHkzNUF/Nt6t3JYowuyB2RV0ghsZd5YAS4SECYFRKKB9f9edSwlD0p6waRzWFIZbVFSCVj/i9xqPc6lb4lxlmNp6BnvNYuJ8H6fiIbwQEYTkLttceHWac6Uxhh7p8InMKaaNNhPme2ctdrTipd4cZ7tjOFWJrLcHfmR32Z3kLoiLHKj3/hEWPmkTjAb8lx/6Ol/IvEVSCCxhsxwIvlR5ktO1MYZeMRj95iK666HqDZQUyGwWOVRg+SMuj37+FMfSK/xq/hVK0ibAwNchf9B8jF9/+SkQUBptMpTs8MXxVzhoLfCpxAaPP/pPqSqTf/LAM7xZneDqqVHGny1h9hRmJ0KEN1ZTVq2HaLZ31c7mdhX6rjD5+dQSn0su8I39R/Bnitiug+gH6FbrXb5p96Bam62d18pkLg5WgTpSRO/zJRRSkCvXyCddLhWn6T69h+o1PgiRwmlFBHWDZn8wMRpCYAtBSvqEKWBkCNlooXq93ZVwY1k09wsefOwSM6kqC8EQ836J33jhQyTnTUpvhYz94DL4/qCr7qavolKaXKdHYm2E9pTDb+aO05xx+cncKSbM9343G8rg25XDnK8Mk1rWRIvL25IxdmfFRQiM4SHIpGhOOARjAcXhJnP2OkVpshxFvNF3eaM3y4srs9SrKSZrCt3uQBQhXAfh2Kh9Y/RzNt5YxIOZJWbtDSLN5o6nwEIwxPfXD+AsDyaIjSBPNZXi6/b9ZIweWekxZjTpI5lw6vRyFiujORpzSaRvYPZMRDSIscsIUqsmzhVrVxz8iyhipZvjZNBnWIaDg7+bsISBJQwKyR5eqYBQSewV612+bRdyrRul98EXEloLhCHRlokydtGEeCfQICKI1I3Qq2SQ2aklYBqDsPkuRIbQChwutUts+GlWulncFZPkmsYte0QbG7ed+HUQIIMIGWqk0CSMAGuz6UhNeZzqZ2gql65yridKAFz2hzm5Mk6wkSDfULswFXmzjqX14f1Ujxr0jnj8jSe/xj6rzLTZoKoUf3/1p/jmm8ewNkxGXlEUqyHuxWWiRhOjNISaLNEbTXL1pwyS0y1+cd8p/kT2dTxtcDYY4mowxN976bMkzrikFzT7X6shtCbMJ4gcg0tzh/mbU0fwSxEzh9cYSbZ4pniOXy69wKcLJ1l+qECgjev/LXt5yl6aE6/u576TOdTbfHzuRWTb4+Jbk/wX4a/wxalX+fO587ctEny0uMBvPT1BcsVleikPlerdv9h7DGHbBAfGaU+59MdCrNhf+vZodX3x9nYvOykU2hj0YxKmMTib1bsnYkAQkF7UXHlrArsqyV7RWB3Fvot1ZK31nqnCwrYJ0hZ+VnKktM6nMieZNpuAw9c6s/zN7/8C5oaFUxNYNxnBG75mfCnEavnYC5VtK+C9M+Jyzc7FcegOS7qzAQenynw2dY6iNClHimpkcbo2SuKKRXJNkz1VQ1QbqE53MMASLv2hBN0Rk/z+Kp+ePsvHM2cYNSRrUcRSUOB8bxTrqsPQqZDkQgd95gJKacx0CtNxKPYmcJoJ2hMGV7NFmnmHDxcuMmk2uc+uUUyt3FLp/1bf4lx/lLdGx+F2zrj3IkGIU5UslfNcHX73YrURuwnDPr7vXneg3nVcE9WteNnkoDFTP2/RK0nMlIcRR8XeFaH05s5FEGmNIQTGZuqdNvSg1fEuTCjRWmO3FM6GQWpZU3y9hmz1UGtlwh92vmRIItcgcgUlp8OE2SIjB89vsT9E4rJNalmTXgmwqzd6WMl+hCzX0Z43mE+3iTsygxr5PMED++iUbKpPhPzsI69zLDkwA7wQCP6bq3+SC+US+kSWkTdDZKAJhlPokTTtSRs/L+mNaoI5j2ymzZ+ae5kH3UVe6BzgX6xMMN8oUjtbxGpIxl4LSV2oIeqt6y07le8jwhBrsUK2lSa15JJadelnHP7hwz/FP5v4CHNDFX6ydAZLDD7jK4t/deEpOhdy5M4LdGuX9IQJQ+wm+BWHNT+z3Vdz99k0CDSGiujJ4YH9+9I6qtH6kdPRjaEi4X0zdAs2iz8pGTpS5gsT58mI3Tc5bgWq55E8X8Etpzj7ZI4X73cZNjpMGDAsu4QHeyx/LEPxrIu7vIr2d8/ORQchmbM17GYGq+Ej1qqDrrpvb1F8E8KyEZY5KDA/ZtIdVxxOrlKU0FCaBQUv1vZRPBuRvtxGNnuI7k0NEqNoICpBsK2JOXdEXEQ2TfWYS3dM8BMPnuLvjT2Lr0NaGi4Gw7z1xiz505LcpT7JkyvobIrWkQJ+VrLxuKI0V+bpkQX+8+Fvk5MR+c087n+9NsprrxwksSKZ+14Hc6MN6xubqaU3uNYSWXW7sDAoTcgIgUynSa0epTmT4+TBDJ0HbRxj8PB7oUX0fIGDf9QceJE1d4e4aKWwmhqnJqn6KRQKg90Z274tmwklFLI0D2UQCrIdD9HzoP+jpaOLXJbqsQTdUcHTHzrFfzf5VZICkvKHe4/tRbTvE52/hLBsEsuP87o3w5y9zpRZYdhQHJ9e5LVoGqOfIGGagw61uwUVEZ06h3laoLV+1zbtN3PtrNkbcWjPhWTGWtznrJCTLothyEl/govVIaZPVolOn39f37kd3BFx0UmX9jQEUz4HkuVNo0mB1IPe2eawR7ubJEza9ErTBClBexrCtKawr8aDQyscTa6QkRGBhm/2SiwFBV6anyVzSZLYUJi1LqLTQ71fG2k9qFh1yx7KTIAwmFcT6M15VkQwfFUhG11Eu7t7OgX6PunVEKFN5msFlqOIjAjJSfsWL605u8yhiXXOqVGCYhIzkxm4+N6DSQ3CNJH5HMJxiMaL+EMunXGL+hEweoLkShaz3hyMifdxf8I0wTCQ2Swik6J3sETjIESjPvelV0mKQYX+NSKt8bQiigQy0oNiTLV3ajveFa0wu/BGaxo/ZXHcWcfTgtPro9jnE6RWd3Ezv/cbipUGYmyYcDhDc8akNLXBgcIGrgioKY/fbT7B784/iHcuh+gu3tlr/jG5I+ISlNLse3qBX5l8kSfcecDEYJB6OG3W+b8ee4mL+0rXi/tKdpuPZc6SN7oMyR5JEZGSgrSwOR3A3zrzOapLeYafNxj+5pVBPnyrPUjb+wCDUfs+8s0LZE6ZZByHCffWlaZud1DtTWv+XZKGHDWaJL57hlQqyflDc/zg6CwzVpVHnA7WTTuYTyXXeOLgv+Nflx7n9+Y+QWljDFGuEm1UtvHqfzRkOkV4eIp+wWb5Iyb2fU0Ol+b5r8ef5cXOAb6++FGG17MIreGHxb2lgUynEK5L/8gE9TmX5kH44mee5cn0JR60V8lJ+5ZmYW0d0NKCKDSQgUYGsbAAaKVJlDXfvXCQK6NFnk6dp6scxIs55r68jG51iHbTruVHQNoWzYeHqd5nED3Y5h8e+xLDsouvDc4EKf7lix9h/5c1I+XmoDfQDubO7FwMwZDbYZ+1QUZG13+NBFwRsd9ZJ2n4JGWfjOwxZLY5Zm2QkgJPawIN5UhyRQte92aprm+m7q0HROsbP5Z1y24r0vqhaI1qtdC9HmbnABthlrzRRen2DSsTIC0ccqZkxt4gcjazd+7V1FDHwR9y6JYMgtGAx0eXeTw3z4fdQZ+W38t/DJVPDVJhlX7vPj+WDYUsKmHTGRv44PVHAo6n5nnQXqUoJYYQBDrC0xGe1iyGCdajDKpjYnoaGWxPKuhORIYa5Rl0+jaeGniNWW1Qq+sDa569/Jw2k0T8rMQbVswUmsyZbVJC8kY/wdWgiFkzca+sIzo9ovc4t9kJ3Jkzl1BR9tJc7I8wbHQYNbgegjGk4kOJeTxtYKEwhKYcJfhS8xHW+xler02x1szQqSZwF2ysNsyeCbCrHazlKmHsavwjY3XgB/X9tDIujzrrpG/62TVXVnUbG/l7DT1aZOEzgvR0nT85c4bP5t5k2OiQFBZjZoP2wx5Xk1kML4vZG+e9MogjFzozCpUJGZso8+mRefa5GzzhLJGXgxbHMEhU+U73fi50R/n6+aNEVYfRHwjSby6hux5R3CTsHSgkkZYIpdFBuHtC0T8CwrIxhgroYo7aMc2Tj5/jydwVXCFYjeAfLn2Ss+URsheA9Q2U5+/453VnxEVDq2+zEWbo6Bu/4lrB3uz1+WsgOHUV8UZzisV2noUrJeyySXEBSq+1MNo+LK+h2p1BNlgsLD8yhqdZauco2F0C/c5Ga9cq+MU9/oijjMv4oTK/OPU6z6TO8IAtuDbWMiJgdqLCPEPovgGBQLy9b/ZN6ETEY4eucF9mjU9kTvFx99pqMXHL81uNsrzRmuZMbQTrZJL8iiZ/pkU4v3AH73R3IBSbhX57N3woLBOdThLmEuhRn18ovca0VcESkoayObM+Sn8+zdB6RNRo3hPz4B0RF3O9SfO7Y/xvY5/k38w8wZHSOjnLYypRQ6Jphi6+sjjbHGGxmsdr2zgLNkZPMLSucZqKxEaAWW6C56P6AVrt/Ie5k9FKkywrls8N055xaAxbTGz3RW0DGan45OhZzqVH8CILL3rvmp606fORwgUmrRoTRgtwCHREQ/Xpanjdn2C+X+K3Fo+z/toodkNQOhPiVAKMcoPd79D2o2FIRUb2SEqf7qhAPngYWW0RLi7dExPnViPHRlj66XF6o5rH9p9jzl6nGqX5jeY0LzX3o9/MUrqoSM537plS3TsiLurKIvv+ZRcSLvXHRjm7L08/r+mPByA0omMifUH+lGDfy3VErwuNFoThIEMpDCGKrtet7MXBtuVoRfpKh+FEhqqXZ/2BNEfZY+dPDJp5/eXia6iCJnofr6mBQG4W/FmbTel8HbIQOZSjDP/HytOcXR/BeDnDoS8toVttdLuDDsMb4zfmHUihKUoPWyi86YDK8Ty5Sy7G6vo9maH44+LPFhn6+UV+dvxNPpY8x1ELftPP8C+vPM3aUoFDf9RFvnz6njqXujMH+lGEarYQ/T6JcpHQlZhdgQhtEBqjJzD6kFoLkZUm2u+jO51B3DU+U7ljCC/EbivMrkE5ytJQdZKbocrdhPRDVtbzfMs9slkke5687DNuDLK6XGHeksxwO64d0AdoIqWIgJZStLTFcjjC8+2DrHhZTixMIFYdMmsa3Wyh2p1BgVw8hm9L6AqcrM9wojNovIaASCBDPWgqtkcRoabdt9kIMgTawBCKRpSiUk9j1E2MTmdgbHkPcWc8TlSE6nkI38d+4zKl8wkwDbRjDyqmowgRKXS7S9Rs3ppSHL+UdwatkfUWqQWT7kiG7zYOY4mQx5wlpszdJS7GYpmJ39vHcmkf/+MjU/zOgYd5augK/1XpeXKbzbx+GOUo5GwwREc5dJSNp22ebxzgZGWMjXKWzOsOTl2zf97HXqsimh2iRnPbTALvBYQUdMYF/9HRHwwyRoVmOXKw6gaZRQ9zo/2+Hal3G2atR/XNYX6jkqH4SIdHnLOc7o5jnU6SXNXIRuee64Jz5wy0VIRWENVqUKvdsV8T8/7Rfh/Z8rA6aebbRc6544wYLRzRBAZH3uUwA4rN/+7NSVJ3e6SvdHBqDr0Rh/lMkZzdo14EiwBHmO8w7ww2zRIDrYjQlJXDeX+MrrJphAm6yuZ0dZTySg571aJwNsDd8JBXVonK5e24zXsPIYlczVF3iRFjYBvvaQvpC4xOgHi/BdG7ENEPcDYEPcvhQm+EcvZNlrs5nCq4NQX34LPZJe6MMe8H3ekgVEThTZPyv53hy+lZfiP5k6ibFvNWEyZebSDWKuh71BVa9TyMpQ2MisO4KNG74HJ23yF+8Yn/mIlsk784/V2ecJfJCElSWiyHPl/vHGEtyPFafZrldpaNSgb7ioP0BVYXZABWSzPVUljNPu6V6sAh4h59RtuCVtgNwe9WHmXcbfBQ8irLQQGzByKINpuz3Wvr8y2iXGXsBxmCrMXzlx7l08OPklrSjLwxyJhVzXuvv1IsLnsI1e0OKtIrVYZOvEf67fv0QNqp6KBPuLIKgL24gmOZ5O4/wEovz6XRHM/lD3LAKiONgCSwFiX4w41jLDQLVC4WSaxKRhYUxVc2EO0eqlJ9RxOre/n5bCd2U/Pa+iSLqTwA1SCFcU1c9rBFTlSrIb9fwwFGbt5V38PvYiwue5U9ci5w7SxP1jtkriax2pLfKjzGN0ePkLADElZApZOkcyWH2RHklgVuVZFcCxCtLtrz7qkMnZ2MVprMUsjKG0WabpFzhTEIJFML0fVnHcOuGWuxuMTsblSEVhHqyiLZtQ2yhsHo11KDzodCoKVkOupCvzFILAkCCEJ0GBL1+4P6ql3iM7ftqIjEd89w8JXkILHHNAfmoc0WUbc7eNa7ZGKNicUlZo+ggz7RtfqJOMFk21CtFqp1750fxHxw7n0jqZiYmJiYHUcsLjExMTExW04sLjExMTExW04sLjExMTExW04sLjExMTExW47QOs79i4mJiYnZWuKdS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlhOLS0xMTEzMlvP/Bzn9k6ptNpvaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 500x200 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 2행 5열의 서브플롯 생성\n",
    "fig, axs = plt.subplots(2, 5, figsize=(5, 2))\n",
    "\n",
    "# 각 서브플롯에 이미지를 표시\n",
    "for i in range(2):\n",
    "    for j in range(5):\n",
    "        index = i * 5 + j  # 현재 그리드의 인덱스 계산\n",
    "        axs[i, j].imshow(train_x[index])  # 해당 위치의 서브플롯에 이미지 표시\n",
    "        axs[i, j].axis('off')  # 축 눈금 숨기기\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "97dee197",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74d6ee3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4], dtype=uint8)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2b5e6a0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_scaled = train_x.reshape(-1, 1, 28, 28) / 255\n",
    "test_scaled = test_x.reshape(-1, 1, 28, 28) / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7eb8968a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Affine:\n",
    "    \n",
    "    def __init__(self, W, b):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        \n",
    "        self.x = None\n",
    "        self.original_x_shape = None\n",
    "        # 가중치와 편향 매개변수의 미분\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        # 텐서 대응\n",
    "        self.original_x_shape = x.shape\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        self.x = x\n",
    "\n",
    "        out = np.dot(self.x, self.W) + self.b\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = np.dot(dout, self.W.T)\n",
    "        self.dW = np.dot(self.x.T, dout)\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "        \n",
    "        dx = dx.reshape(*self.original_x_shape)  # 입력 데이터 모양 변경(텐서 대응)\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f33070e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftmaxWithLoss:\n",
    "    def __init__(self):\n",
    "        self.loss = None # 손실함수\n",
    "        self.y = None    # softmax의 출력\n",
    "        self.t = None    # 정답 레이블(원-핫 인코딩 형태)\n",
    "        \n",
    "    def forward(self, x, t):\n",
    "        self.t = t\n",
    "        self.y = softmax(x)\n",
    "        self.loss = cross_entropy_error(self.y, self.t)\n",
    "        \n",
    "        return self.loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        batch_size = self.t.shape[0]\n",
    "        if self.t.size == self.y.size: # 정답 레이블이 원-핫 인코딩 형태일 때\n",
    "            dx = (self.y - self.t) / batch_size\n",
    "        else:\n",
    "            dx = self.y.copy()\n",
    "            dx[np.arange(batch_size), self.t] -= 1\n",
    "            dx = dx / batch_size\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6f4a3795",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adam:\n",
    "\n",
    "    def __init__(self, lr=0.001, beta1=0.9, beta2=0.999):\n",
    "        self.lr = lr\n",
    "        self.beta1 = beta1\n",
    "        self.beta2 = beta2\n",
    "        self.iter = 0\n",
    "        self.m = None\n",
    "        self.v = None\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        if self.m is None:\n",
    "            self.m, self.v = {}, {}\n",
    "            for key, val in params.items():\n",
    "                self.m[key] = np.zeros_like(val)\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "        \n",
    "        self.iter += 1\n",
    "        lr_t  = self.lr * np.sqrt(1.0 - self.beta2**self.iter) / (1.0 - self.beta1**self.iter)         \n",
    "        \n",
    "        for key in params.keys():\n",
    "            self.m[key] += (1 - self.beta1) * (grads[key] - self.m[key])\n",
    "            self.v[key] += (1 - self.beta2) * (grads[key]**2 - self.v[key])\n",
    "            \n",
    "            params[key] -= lr_t * self.m[key] / (np.sqrt(self.v[key]) + 1e-7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "064c6be2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RMSprop:\n",
    "\n",
    "    def __init__(self, lr=0.01, decay_rate = 0.99):\n",
    "        self.lr = lr\n",
    "        self.decay_rate = decay_rate\n",
    "        self.h = None\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        if self.h is None:\n",
    "            self.h = {}\n",
    "            for key, val in params.items():\n",
    "                self.h[key] = np.zeros_like(val)\n",
    "            \n",
    "        for key in params.keys():\n",
    "            self.h[key] *= self.decay_rate\n",
    "            self.h[key] += (1 - self.decay_rate) * grads[key] * grads[key]\n",
    "            params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0cc93d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaGrad:\n",
    "\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "        self.h = None\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        if self.h is None:\n",
    "            self.h = {}\n",
    "            for key, val in params.items():\n",
    "                self.h[key] = np.zeros_like(val)\n",
    "            \n",
    "        for key in params.keys():\n",
    "            self.h[key] += grads[key] * grads[key]\n",
    "            params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dbb378e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Momentum:\n",
    "\n",
    "    def __init__(self, lr=0.01, momentum=0.9):\n",
    "        self.lr = lr\n",
    "        self.momentum = momentum\n",
    "        self.v = None\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        if self.v is None:\n",
    "            self.v = {}\n",
    "            for key, val in params.items():                                \n",
    "                self.v[key] = np.zeros_like(val)\n",
    "                \n",
    "        for key in params.keys():\n",
    "            self.v[key] = self.momentum*self.v[key] - self.lr*grads[key] \n",
    "            params[key] += self.v[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "206d9f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD:\n",
    "\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        for key in params.keys():\n",
    "            params[key] -= self.lr * grads[key] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9ce042db",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Nesterov:\n",
    "    \n",
    "    def __init__(self, lr=0.01, momentum=0.9):\n",
    "        self.lr = lr\n",
    "        self.momentum = momentum\n",
    "        self.v = None\n",
    "        \n",
    "    def update(self, params, grads):\n",
    "        if self.v is None:\n",
    "            self.v = {}\n",
    "            for key, val in params.items():\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "            \n",
    "        for key in params.keys():\n",
    "            self.v[key] *= self.momentum\n",
    "            self.v[key] -= self.lr * grads[key]\n",
    "            params[key] += self.momentum * self.momentum * self.v[key]\n",
    "            params[key] -= (1 + self.momentum) * self.lr * grads[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1121d96e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    if x.ndim == 2:\n",
    "        x = x.T\n",
    "        x = x - np.max(x, axis=0)\n",
    "        y = np.exp(x) / np.sum(np.exp(x), axis=0)\n",
    "        return y.T "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "461d785d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy_error(y, t):\n",
    "    if y.ndim == 1:\n",
    "        t = t.reshape(1, t.size)\n",
    "        y = y.reshape(1, y.size)\n",
    "        \n",
    "    # 훈련 데이터가 원-핫 벡터라면 정답 레이블의 인덱스로 반환\n",
    "    if t.size == y.size:\n",
    "        t = t.argmax(axis=1)\n",
    "             \n",
    "    batch_size = y.shape[0]\n",
    "    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0dc410e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def im2col(input_data, filter_h, filter_w, stride=1, pad=0):\n",
    "    \"\"\"다수의 이미지를 입력받아 2차원 배열로 변환한다(평탄화).\n",
    "    Parameters\n",
    "    ----------\n",
    "    input_data : 4차원 배열 형태의 입력 데이터(이미지 수, 채널 수, 높이, 너비)\n",
    "    filter_h : 필터의 높이\n",
    "    filter_w : 필터의 너비\n",
    "    stride : 스트라이드\n",
    "    pad : 패딩\n",
    "    Returns\n",
    "    -------\n",
    "    col : 2차원 배열\n",
    "    \"\"\"\n",
    "    N, C, H, W = input_data.shape\n",
    "    \n",
    "    out_h = (H + 2*pad - filter_h)//stride + 1\n",
    "    out_w = (W + 2*pad - filter_w)//stride + 1\n",
    "    \n",
    "    \n",
    "    img = np.pad(input_data, [(0,0), (0,0), (pad, pad), (pad, pad)], 'constant')\n",
    "    col = np.zeros((N, C, filter_h, filter_w, out_h, out_w))\n",
    "    \n",
    "    for y in range(filter_h):\n",
    "        y_max = y + stride*out_h\n",
    "        for x in range(filter_w):\n",
    "            x_max = x + stride*out_w\n",
    "            col[:, :, y, x, :, :] = img[:, :, y:y_max:stride, x:x_max:stride]\n",
    "            \n",
    "            \n",
    "    col = col.transpose(0, 4, 5, 1, 2, 3).reshape(N*out_h*out_w, -1)# 2차원으로 바꾸어 반환\n",
    "    return col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "26e8fce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def col2im(col, input_shape, filter_h, filter_w, stride=1, pad=0):\n",
    "    \"\"\"(im2col과 반대) 2차원 배열을 입력받아 다수의 이미지 묶음으로 변환한다.\n",
    "    Parameters\n",
    "    ----------\n",
    "    col : 2차원 배열(입력 데이터)\n",
    "    input_shape : 원래 이미지 데이터의 형상（예：(10, 1, 28, 28)）\n",
    "    filter_h : 필터의 높이\n",
    "    filter_w : 필터의 너비\n",
    "    stride : 스트라이드\n",
    "    pad : 패딩\n",
    "    Returns\n",
    "    -------\n",
    "    img : 변환된 이미지들\n",
    "    \"\"\"\n",
    "    N, C, H, W = input_shape\n",
    "    out_h = (H + 2*pad - filter_h)//stride + 1\n",
    "    out_w = (W + 2*pad - filter_w)//stride + 1\n",
    "    col = col.reshape(N, out_h, out_w, C, filter_h, filter_w).transpose(0, 3, 4, 5, 1, 2)\n",
    "    img = np.zeros((N, C, H + 2*pad + stride - 1, W + 2*pad + stride - 1))\n",
    "    for y in range(filter_h):\n",
    "        y_max = y + stride*out_h\n",
    "        for x in range(filter_w):\n",
    "            x_max = x + stride*out_w\n",
    "            img[:, :, y:y_max:stride, x:x_max:stride] += col[:, :, y, x, :, :]\n",
    "    return img[:, :, pad:H + pad, pad:W + pad]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5520c6b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Convolution:\n",
    "    def __init__(self, W, b, stride=1, pad=0):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "        # 중간 데이터（backward 시 사용）\n",
    "        self.x = None\n",
    "        self.col = None\n",
    "        self.col_W = None\n",
    "        # 가중치와 편향 매개변수의 기울기\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "    def forward(self, x):\n",
    "        FN, C, FH, FW = self.W.shape\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = 1 + int((H + 2*self.pad - FH) / self.stride)\n",
    "        out_w = 1 + int((W + 2*self.pad - FW) / self.stride)\n",
    "        col = im2col(x, FH, FW, self.stride, self.pad)\n",
    "        col_W = self.W.reshape(FN, -1).T\n",
    "        out = np.dot(col, col_W) + self.b\n",
    "        out = out.reshape(N, out_h, out_w, -1).transpose(0, 3, 1, 2)\n",
    "        self.x = x\n",
    "        self.col = col\n",
    "        self.col_W = col_W\n",
    "        return out\n",
    "    def backward(self, dout):\n",
    "        FN, C, FH, FW = self.W.shape\n",
    "        dout = dout.transpose(0,2,3,1).reshape(-1, FN)\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "        self.dW = np.dot(self.col.T, dout)\n",
    "        self.dW = self.dW.transpose(1, 0).reshape(FN, C, FH, FW)\n",
    "        dcol = np.dot(dout, self.col_W.T)\n",
    "        dx = col2im(dcol, self.x.shape, FH, FW, self.stride, self.pad)\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c9fb122c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Pooling:\n",
    "    def __init__(self, pool_h, pool_w, stride=1, pad=0):\n",
    "        self.pool_h = pool_h\n",
    "        self.pool_w = pool_w\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "        self.x = None\n",
    "        self.arg_max = None\n",
    "        \n",
    "    def forward(self, x):\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = int(1 + (H - self.pool_h) / self.stride)\n",
    "        out_w = int(1 + (W - self.pool_w) / self.stride)\n",
    "        col = im2col(x, self.pool_h, self.pool_w, self.stride, self.pad)\n",
    "        col = col.reshape(-1, self.pool_h*self.pool_w)\n",
    "        arg_max = np.argmax(col, axis=1)\n",
    "        out = np.max(col, axis=1)\n",
    "        out = out.reshape(N, out_h, out_w, C).transpose(0, 3, 1, 2)\n",
    "        self.x = x\n",
    "        self.arg_max = arg_max\n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dout = dout.transpose(0, 2, 3, 1)\n",
    "        pool_size = self.pool_h * self.pool_w\n",
    "        dmax = np.zeros((dout.size, pool_size))\n",
    "        dmax[np.arange(self.arg_max.size), self.arg_max.flatten()] = dout.flatten()\n",
    "        dmax = dmax.reshape(dout.shape + (pool_size,))\n",
    "        dcol = dmax.reshape(dmax.shape[0] * dmax.shape[1] * dmax.shape[2], -1)\n",
    "        dx = col2im(dcol, self.x.shape, self.pool_h, self.pool_w, self.stride, self.pad)\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e6dbdf1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Trainer:\n",
    "    def __init__(self, network, x_train, t_train, x_test, t_test,\n",
    "                 epochs=20, mini_batch_size=100,\n",
    "                 optimizer='SGD', optimizer_param={'lr':0.01},\n",
    "                 evaluate_sample_num_per_epoch=None, verbose=True):\n",
    "        \n",
    "        self.network = network\n",
    "        self.verbose = verbose\n",
    "        self.x_train = x_train\n",
    "        self.t_train = t_train\n",
    "        self.x_test = x_test\n",
    "        self.t_test = t_test\n",
    "        self.epochs = epochs\n",
    "        self.batch_size = mini_batch_size\n",
    "        self.evaluate_sample_num_per_epoch = evaluate_sample_num_per_epoch\n",
    "        \n",
    "        # optimzer\n",
    "        \n",
    "        optimizer_class_dict = {'sgd':SGD, 'momentum':Momentum, 'nesterov':Nesterov,\n",
    "                                'adagrad':AdaGrad, 'rmsprpo':RMSprop, 'adam':Adam}\n",
    "        self.optimizer = optimizer_class_dict[optimizer.lower()](**optimizer_param)\n",
    "        self.train_size = x_train.shape[0]\n",
    "        self.iter_per_epoch = max(self.train_size / mini_batch_size, 1)\n",
    "        self.max_iter = int(epochs * self.iter_per_epoch)\n",
    "        self.current_iter = 0\n",
    "        self.current_epoch = 0\n",
    "        self.train_loss_list = []\n",
    "        self.train_acc_list = []\n",
    "        self.test_acc_list = []\n",
    "    \n",
    "    def train_step(self):\n",
    "        \n",
    "        batch_mask = np.random.choice(self.train_size, self.batch_size)\n",
    "        x_batch = self.x_train[batch_mask]\n",
    "        t_batch = self.t_train[batch_mask]\n",
    "        \n",
    "        grads = self.network.gradient(x_batch, t_batch)\n",
    "        self.optimizer.update(self.network.params, grads)\n",
    "        loss = self.network.loss(x_batch, t_batch)\n",
    "        self.train_loss_list.append(loss)\n",
    "        if self.verbose: \n",
    "            print(\"train loss:\" + str(loss))\n",
    "        if self.current_iter % self.iter_per_epoch == 0:\n",
    "            self.current_epoch += 1\n",
    "            x_train_sample, t_train_sample = self.x_train, self.t_train\n",
    "            x_test_sample, t_test_sample = self.x_test, self.t_test\n",
    "            if not self.evaluate_sample_num_per_epoch is None:\n",
    "                t = self.evaluate_sample_num_per_epoch\n",
    "                x_train_sample, t_train_sample = self.x_train[:t], self.t_train[:t]\n",
    "                x_test_sample, t_test_sample = self.x_test[:t], self.t_test[:t]\n",
    "            train_acc = self.network.accuracy(x_train_sample, t_train_sample)\n",
    "            test_acc = self.network.accuracy(x_test_sample, t_test_sample)\n",
    "            self.train_acc_list.append(train_acc)\n",
    "            self.test_acc_list.append(test_acc)\n",
    "            if self.verbose: print(\"epoch:\" + str(self.current_epoch) + \", train acc:\" + str(train_acc) + \", test acc:\" + str(test_acc) + \" ===\")\n",
    "        self.current_iter += 1\n",
    "    def train(self):\n",
    "        for i in range(self.max_iter):\n",
    "            self.train_step()\n",
    "        test_acc = self.network.accuracy(self.x_test, self.t_test)\n",
    "        if self.verbose:\n",
    "            print(\"Final Test Accuracy\")\n",
    "            print(\"test acc:\" + str(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cc54a8e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet:\n",
    "    def __init__(self, input_dim=(1, 28, 28),\n",
    "                 conv_param={'filter_num':30, 'filter_size':5, 'pad':0, 'stride':1},\n",
    "                 hidden_size=100, output_size=10, weight_init_std=0.01):\n",
    "        filter_num = conv_param['filter_num']\n",
    "        filter_size = conv_param['filter_size']\n",
    "        filter_pad = conv_param['pad']\n",
    "        filter_stride = conv_param['stride']\n",
    "        input_size = input_dim[1]\n",
    "        conv_output_size = (input_size - filter_size + 2*filter_pad) / filter_stride + 1\n",
    "        pool_output_size = int(filter_num * (conv_output_size/2) * (conv_output_size/2))\n",
    "        \n",
    "        # 가중치 초기화\n",
    "        self.params = {}\n",
    "        self.params['W1'] = weight_init_std * \\\n",
    "                            np.random.randn(filter_num, input_dim[0], filter_size, filter_size)\n",
    "        self.params['b1'] = np.zeros(filter_num)\n",
    "        self.params['W2'] = weight_init_std * \\\n",
    "                            np.random.randn(pool_output_size, hidden_size)\n",
    "        self.params['b2'] = np.zeros(hidden_size)\n",
    "        self.params['W3'] = weight_init_std * \\\n",
    "                            np.random.randn(hidden_size, output_size)\n",
    "        self.params['b3'] = np.zeros(output_size)\n",
    "        \n",
    "        # 계층 생성\n",
    "        self.layers = OrderedDict()\n",
    "        self.layers['Conv1'] = Convolution(self.params['W1'], self.params['b1'],\n",
    "                                           conv_param['stride'], conv_param['pad'])\n",
    "        self.layers['Relu1'] = Relu()\n",
    "        self.layers['Pool1'] = Pooling(pool_h=2, pool_w=2, stride=2)\n",
    "        self.layers['Affine1'] = Affine(self.params['W2'], self.params['b2'])\n",
    "        self.layers['Relu2'] = Relu()\n",
    "        self.layers['Affine2'] = Affine(self.params['W3'], self.params['b3'])\n",
    "        self.last_layer = SoftmaxWithLoss()\n",
    "        \n",
    "    def predict(self, x):\n",
    "        for layer in self.layers.values():\n",
    "            x = layer.forward(x)\n",
    "        return x\n",
    "    \n",
    "    def loss(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        return self.last_layer.forward(y, t)\n",
    "    \n",
    "    def accuracy(self, x, t, batch_size=100):\n",
    "        if t.ndim != 1 : t = np.argmax(t, axis=1)\n",
    "        acc = 0.0\n",
    "        for i in range(int(x.shape[0] / batch_size)):\n",
    "            tx = x[i*batch_size:(i+1)*batch_size]\n",
    "            tt = t[i*batch_size:(i+1)*batch_size]\n",
    "            y = self.predict(tx)\n",
    "            y = np.argmax(y, axis=1)\n",
    "            acc += np.sum(y == tt)\n",
    "        return acc / x.shape[0]\n",
    "    \n",
    "    def gradient(self, x, t):\n",
    "        # forward\n",
    "        self.loss(x, t)\n",
    "        # backward\n",
    "        dout = 1\n",
    "        dout = self.last_layer.backward(dout)\n",
    "        layers = list(self.layers.values())\n",
    "        layers.reverse()\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout)\n",
    "            \n",
    "        # 결과 저장\n",
    "        grads = {}\n",
    "        grads['W1'], grads['b1'] = self.layers['Conv1'].dW, self.layers['Conv1'].db\n",
    "        grads['W2'], grads['b2'] = self.layers['Affine1'].dW, self.layers['Affine1'].db\n",
    "        grads['W3'], grads['b3'] = self.layers['Affine2'].dW, self.layers['Affine2'].db\n",
    "        return grads\n",
    "    \n",
    "    def save_params(self, file_name=\"params.pkl\"):\n",
    "        params = {}\n",
    "        for key, val in self.params.items():\n",
    "            params[key] = val\n",
    "        with open(file_name, 'wb') as f:\n",
    "            pickle.dump(params, f)\n",
    "            \n",
    "    def load_params(self, file_name=\"params.pkl\"):\n",
    "        with open(file_name, 'rb') as f:\n",
    "            params = pickle.load(f)\n",
    "            \n",
    "        for key, val in params.items():\n",
    "            self.params[key] = val\n",
    "            \n",
    "        for i, key in enumerate(['Conv1', 'Affine1', 'Affine2']):\n",
    "            self.layers[key].W = self.params['W' + str(i+1)]\n",
    "            self.layers[key].b = self.params['b' + str(i+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9f8ca64f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Relu:\n",
    "    def __init__(self):\n",
    "        self.mask = None\n",
    "    def forward(self, x):\n",
    "        self.mask = (x <= 0)\n",
    "        out = x.copy()\n",
    "        out[self.mask] = 0\n",
    "        return out\n",
    "    def backward(self, dout):\n",
    "        dout[self.mask] = 0\n",
    "        dx = dout\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f4eb7a2",
   "metadata": {},
   "source": [
    "# 무지성 아담"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2f9f704e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:2.260967045489168\n",
      "epoch:1, train acc:0.10218333333333333, test acc:0.101 ===\n",
      "train loss:2.151248811980249\n",
      "train loss:1.9802502620333047\n",
      "train loss:1.674143092873105\n",
      "train loss:1.3823010277175412\n",
      "train loss:1.3135975925104415\n",
      "train loss:0.9938284158139195\n",
      "train loss:1.0006987852950642\n",
      "train loss:0.9217491040675072\n",
      "train loss:0.9572273255314013\n",
      "train loss:0.9568945031121504\n",
      "train loss:0.8306772293259296\n",
      "train loss:0.7858682493108172\n",
      "train loss:0.7312785207846979\n",
      "train loss:0.5987000127117191\n",
      "train loss:0.754699143963161\n",
      "train loss:0.4174378747257715\n",
      "train loss:0.604815205423305\n",
      "train loss:0.43822591064169186\n",
      "train loss:0.4530920587986876\n",
      "train loss:0.4893034174035493\n",
      "train loss:0.4009352856423002\n",
      "train loss:0.548170315170409\n",
      "train loss:0.43808908228695925\n",
      "train loss:0.33144057692445217\n",
      "train loss:0.5408690914003877\n",
      "train loss:0.19435530181319596\n",
      "train loss:0.2304802116157132\n",
      "train loss:0.29749861382293935\n",
      "train loss:0.2955119980500268\n",
      "train loss:0.34264080027455607\n",
      "train loss:0.222831246030167\n",
      "train loss:0.20948426237899853\n",
      "train loss:0.23988164111554025\n",
      "train loss:0.2408252917189189\n",
      "train loss:0.37699794644206064\n",
      "train loss:0.2682311738553349\n",
      "train loss:0.28972094085158323\n",
      "train loss:0.3176286003175942\n",
      "train loss:0.27016905465353813\n",
      "train loss:0.3439750688702637\n",
      "train loss:0.13195759244162886\n",
      "train loss:0.20902702115302657\n",
      "train loss:0.18721477049453678\n",
      "train loss:0.19363903154595932\n",
      "train loss:0.1919168463319618\n",
      "train loss:0.18235652752354622\n",
      "train loss:0.1168507261104828\n",
      "train loss:0.20483085109882648\n",
      "train loss:0.1471869373612301\n",
      "train loss:0.3407856521797034\n",
      "train loss:0.3096854197656571\n",
      "train loss:0.25812358430195076\n",
      "train loss:0.12224458067103408\n",
      "train loss:0.2820165577258765\n",
      "train loss:0.08209269465908693\n",
      "train loss:0.202427937848391\n",
      "train loss:0.3350083229936023\n",
      "train loss:0.20081599282461024\n",
      "train loss:0.24740806906073842\n",
      "train loss:0.17139643184919895\n",
      "train loss:0.16354188610536466\n",
      "train loss:0.08676507520538575\n",
      "train loss:0.08529463547862394\n",
      "train loss:0.1869780685994909\n",
      "train loss:0.06324561824247629\n",
      "train loss:0.2344570638127344\n",
      "train loss:0.1855177415122656\n",
      "train loss:0.16553945338376788\n",
      "train loss:0.24195579015576268\n",
      "train loss:0.1249280217258834\n",
      "train loss:0.09547256475948825\n",
      "train loss:0.06962041650582936\n",
      "train loss:0.06272475875023531\n",
      "train loss:0.1408272650029296\n",
      "train loss:0.1800068704260488\n",
      "train loss:0.24468683511838918\n",
      "train loss:0.16026420965696203\n",
      "train loss:0.1688541268539126\n",
      "train loss:0.07943293230401216\n",
      "train loss:0.06018651244128552\n",
      "train loss:0.0830323257674568\n",
      "train loss:0.04142631843085367\n",
      "train loss:0.13717609188942956\n",
      "train loss:0.05429984307192079\n",
      "train loss:0.12560434901873122\n",
      "train loss:0.21231165822044365\n",
      "train loss:0.14852897409433627\n",
      "train loss:0.19200594735030052\n",
      "train loss:0.18720429735708066\n",
      "train loss:0.04986413502346251\n",
      "train loss:0.15897652091694384\n",
      "train loss:0.056186640742753494\n",
      "train loss:0.05908338874062362\n",
      "train loss:0.08147085905068054\n",
      "train loss:0.0943586922588668\n",
      "train loss:0.10522344818779232\n",
      "train loss:0.1455775495740511\n",
      "train loss:0.13053109976156782\n",
      "train loss:0.17788189563961074\n",
      "train loss:0.19822192377856795\n",
      "train loss:0.15379108545978773\n",
      "train loss:0.038376792146158606\n",
      "train loss:0.1260760939096796\n",
      "train loss:0.20476457242927698\n",
      "train loss:0.13750985477052274\n",
      "train loss:0.10856818415303579\n",
      "train loss:0.08888618071358373\n",
      "train loss:0.09344000425360177\n",
      "train loss:0.06293001438073183\n",
      "train loss:0.12204557900718037\n",
      "train loss:0.04831798978213351\n",
      "train loss:0.1428157445874594\n",
      "train loss:0.09845734456974013\n",
      "train loss:0.06255782945316994\n",
      "train loss:0.06545898373064028\n",
      "train loss:0.05608079408761217\n",
      "train loss:0.12431483426110168\n",
      "train loss:0.10051264486042834\n",
      "train loss:0.126870999609444\n",
      "train loss:0.09584556923599896\n",
      "train loss:0.06892860860706826\n",
      "train loss:0.07843641726248701\n",
      "train loss:0.09705080945353065\n",
      "train loss:0.03820786283166123\n",
      "train loss:0.11227217692932893\n",
      "train loss:0.08514237999683649\n",
      "train loss:0.20393464972422343\n",
      "train loss:0.11934026620670213\n",
      "train loss:0.1004518958330309\n",
      "train loss:0.12543290552766295\n",
      "train loss:0.15465596667423484\n",
      "train loss:0.045690864605233916\n",
      "train loss:0.06403228932799993\n",
      "train loss:0.14723661256394294\n",
      "train loss:0.033397839513102656\n",
      "train loss:0.1136542190162351\n",
      "train loss:0.08630259600369718\n",
      "train loss:0.1263691548766258\n",
      "train loss:0.08022998392578153\n",
      "train loss:0.04288604907192263\n",
      "train loss:0.06466963361845099\n",
      "train loss:0.13843816991320482\n",
      "train loss:0.22069712445143896\n",
      "train loss:0.07618700892935307\n",
      "train loss:0.11436251107798609\n",
      "train loss:0.06426862530002357\n",
      "train loss:0.05994910217072005\n",
      "train loss:0.09883400877606686\n",
      "train loss:0.12058690933538511\n",
      "train loss:0.027989391339695843\n",
      "train loss:0.04666678602026294\n",
      "train loss:0.10459969579068097\n",
      "train loss:0.0784859426100418\n",
      "train loss:0.07517985647166492\n",
      "train loss:0.030434250954805116\n",
      "train loss:0.03883561421906356\n",
      "train loss:0.06653193671991896\n",
      "train loss:0.032915138856377216\n",
      "train loss:0.2215844023784976\n",
      "train loss:0.11917481957589417\n",
      "train loss:0.04545527960536082\n",
      "train loss:0.12549523595325904\n",
      "train loss:0.15143574687344852\n",
      "train loss:0.2476890615312329\n",
      "train loss:0.11681302222084279\n",
      "train loss:0.041030681275516435\n",
      "train loss:0.13229315927277585\n",
      "train loss:0.04149472679764877\n",
      "train loss:0.03776691625597697\n",
      "train loss:0.0696406703390033\n",
      "train loss:0.06310081764851393\n",
      "train loss:0.10435828600041745\n",
      "train loss:0.14198982132865848\n",
      "train loss:0.054286746732723046\n",
      "train loss:0.1191255241759274\n",
      "train loss:0.06965140392610807\n",
      "train loss:0.11666492390236435\n",
      "train loss:0.06550899676181998\n",
      "train loss:0.04502840651709389\n",
      "train loss:0.03480771724668669\n",
      "train loss:0.04693212738465642\n",
      "train loss:0.07885991618358952\n",
      "train loss:0.041323702001839796\n",
      "train loss:0.05019647849737913\n",
      "train loss:0.07715549855571897\n",
      "train loss:0.0786113243521381\n",
      "train loss:0.03929599923467436\n",
      "train loss:0.08559835182029182\n",
      "train loss:0.09028091254763701\n",
      "train loss:0.039176921083857494\n",
      "train loss:0.03295731613774581\n",
      "train loss:0.03866072712885585\n",
      "train loss:0.1967479857595406\n",
      "train loss:0.11219539795119282\n",
      "train loss:0.012977600842146107\n",
      "train loss:0.10143800426238891\n",
      "train loss:0.06605193129270936\n",
      "train loss:0.10469095582765871\n",
      "train loss:0.05100268486296284\n",
      "train loss:0.09771217193012145\n",
      "train loss:0.031261678743470064\n",
      "train loss:0.10439098050749081\n",
      "train loss:0.09492628615154665\n",
      "train loss:0.046319193080772675\n",
      "train loss:0.020265986927235267\n",
      "train loss:0.08589575236052192\n",
      "train loss:0.07245229689475241\n",
      "train loss:0.1417312648625273\n",
      "train loss:0.09874317768688731\n",
      "train loss:0.08277192415837979\n",
      "train loss:0.16595295779403396\n",
      "train loss:0.08565367687733508\n",
      "train loss:0.13168432942840247\n",
      "train loss:0.027889312648210887\n",
      "train loss:0.02354331379329599\n",
      "train loss:0.08551354914747232\n",
      "train loss:0.11222070775786058\n",
      "train loss:0.042288438552706156\n",
      "train loss:0.025730244132953467\n",
      "train loss:0.08445805494932263\n",
      "train loss:0.0401508891420227\n",
      "train loss:0.06638222378148627\n",
      "train loss:0.12268347090041373\n",
      "train loss:0.11437839771046367\n",
      "train loss:0.07475111602476851\n",
      "train loss:0.021921627339833112\n",
      "train loss:0.07120375509652607\n",
      "train loss:0.04763971136548688\n",
      "train loss:0.07008621660343482\n",
      "train loss:0.03394752800453702\n",
      "train loss:0.026080629567502144\n",
      "train loss:0.11027704225125469\n",
      "train loss:0.05614159274911568\n",
      "train loss:0.045733526474029375\n",
      "train loss:0.02838804161447662\n",
      "train loss:0.03072933112625347\n",
      "train loss:0.04531531075233301\n",
      "train loss:0.05881389446671122\n",
      "train loss:0.02071738747987263\n",
      "train loss:0.19228273941088378\n",
      "train loss:0.08749187105163635\n",
      "train loss:0.03411443707730733\n",
      "train loss:0.035784071418511386\n",
      "train loss:0.06461215328571399\n",
      "train loss:0.08292961147148041\n",
      "train loss:0.03217200990751846\n",
      "train loss:0.12840720776275732\n",
      "train loss:0.026744414897861845\n",
      "train loss:0.041689225073507453\n",
      "train loss:0.05024652166096672\n",
      "train loss:0.06761609161385006\n",
      "train loss:0.08424140646379952\n",
      "train loss:0.04553945929581582\n",
      "train loss:0.0919279653787644\n",
      "train loss:0.06991065679912428\n",
      "train loss:0.04444650613553676\n",
      "train loss:0.045004194845732266\n",
      "train loss:0.05901775390507049\n",
      "train loss:0.09686657172731543\n",
      "train loss:0.11993767380397999\n",
      "train loss:0.02001833483487091\n",
      "train loss:0.028980649025957646\n",
      "train loss:0.028298229895453428\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.04787605999816806\n",
      "train loss:0.04219727076431469\n",
      "train loss:0.12642603863688182\n",
      "train loss:0.05072812603297689\n",
      "train loss:0.033811997526081876\n",
      "train loss:0.02937069008446894\n",
      "train loss:0.039774590604466066\n",
      "train loss:0.024774753215185786\n",
      "train loss:0.060928894023168756\n",
      "train loss:0.08650005314934738\n",
      "train loss:0.0505868084297714\n",
      "train loss:0.04497838342697016\n",
      "train loss:0.0646589906315284\n",
      "train loss:0.07068154453712289\n",
      "train loss:0.0453162781574315\n",
      "train loss:0.06423045004206988\n",
      "train loss:0.05950821058469142\n",
      "train loss:0.028692985600316908\n",
      "train loss:0.021340764174821125\n",
      "train loss:0.04506925878970359\n",
      "train loss:0.020214684095861243\n",
      "train loss:0.058491282047150046\n",
      "train loss:0.02165882210052995\n",
      "train loss:0.02124951473556545\n",
      "train loss:0.028020775106038794\n",
      "train loss:0.011249755901093086\n",
      "train loss:0.054188518584584774\n",
      "train loss:0.04783967372237833\n",
      "train loss:0.08263560937101999\n",
      "train loss:0.058030486885674464\n",
      "train loss:0.0225384248624407\n",
      "train loss:0.11898083026577046\n",
      "train loss:0.051703792505029725\n",
      "train loss:0.06630259053437162\n",
      "train loss:0.048014489063836124\n",
      "train loss:0.06631267157784863\n",
      "train loss:0.15356248741413997\n",
      "train loss:0.06406348999910956\n",
      "train loss:0.021663445297200337\n",
      "train loss:0.018322664304780977\n",
      "train loss:0.04627997078015374\n",
      "train loss:0.024470832480432577\n",
      "train loss:0.13124102575828056\n",
      "train loss:0.11260933392098423\n",
      "train loss:0.01604381010145437\n",
      "train loss:0.06406740572978924\n",
      "train loss:0.0629716886047568\n",
      "train loss:0.029978854048405917\n",
      "train loss:0.08211741900186413\n",
      "train loss:0.02104834679667836\n",
      "train loss:0.08377005557043153\n",
      "train loss:0.06841683174997967\n",
      "train loss:0.13162390504221355\n",
      "train loss:0.03007709457412964\n",
      "train loss:0.07726768019366957\n",
      "train loss:0.052264670745371306\n",
      "train loss:0.014964853087785501\n",
      "train loss:0.09332109673294615\n",
      "train loss:0.03211863530842062\n",
      "train loss:0.0374727815757287\n",
      "train loss:0.07467036010087051\n",
      "train loss:0.047298972743417836\n",
      "train loss:0.048895912966439185\n",
      "train loss:0.18269569110316103\n",
      "train loss:0.0455814030636411\n",
      "train loss:0.04058014025632308\n",
      "train loss:0.0358267105010963\n",
      "train loss:0.017384862262961792\n",
      "train loss:0.03117947754126692\n",
      "train loss:0.06198293011295834\n",
      "train loss:0.04298078612822101\n",
      "train loss:0.032652233191774495\n",
      "train loss:0.02679693727249249\n",
      "train loss:0.050828233910012066\n",
      "train loss:0.05585710899825345\n",
      "train loss:0.03800617566144838\n",
      "train loss:0.019863059996531478\n",
      "train loss:0.05331306021883725\n",
      "train loss:0.12457021200841804\n",
      "train loss:0.06385395456816062\n",
      "train loss:0.04379822530065762\n",
      "train loss:0.013636485303617918\n",
      "train loss:0.10266177803132542\n",
      "train loss:0.06165532286605528\n",
      "train loss:0.0312953465216892\n",
      "train loss:0.04399993225541948\n",
      "train loss:0.0122094119364753\n",
      "train loss:0.09389941729256154\n",
      "train loss:0.03872819868843368\n",
      "train loss:0.06171177382449732\n",
      "train loss:0.05670857082682486\n",
      "train loss:0.052237704735595185\n",
      "train loss:0.07296923564098372\n",
      "train loss:0.013135949763702564\n",
      "train loss:0.09563879439885625\n",
      "train loss:0.015542341689144439\n",
      "train loss:0.07607613943249716\n",
      "train loss:0.0468694164547934\n",
      "train loss:0.023721121864925206\n",
      "train loss:0.024364930552508338\n",
      "train loss:0.07423374464903383\n",
      "train loss:0.035547821861464345\n",
      "train loss:0.013356778836883932\n",
      "train loss:0.026480169647139114\n",
      "train loss:0.03509283731632295\n",
      "train loss:0.017173258757092538\n",
      "train loss:0.016082994176030582\n",
      "train loss:0.03862626996162091\n",
      "train loss:0.05482389650379851\n",
      "train loss:0.043371893550468624\n",
      "train loss:0.019929023633057114\n",
      "train loss:0.02947989707558466\n",
      "train loss:0.06411264243543044\n",
      "train loss:0.056679475863161\n",
      "train loss:0.069967799087019\n",
      "train loss:0.01231405160144808\n",
      "train loss:0.04115360731683973\n",
      "train loss:0.046870282183629604\n",
      "train loss:0.010560741143016588\n",
      "train loss:0.01804140836784839\n",
      "train loss:0.06553236792220571\n",
      "train loss:0.02278132076912852\n",
      "train loss:0.011213081888974564\n",
      "train loss:0.02308393347861006\n",
      "train loss:0.06604503306971803\n",
      "train loss:0.0128839086570349\n",
      "train loss:0.041847383227440096\n",
      "train loss:0.07191722690346931\n",
      "train loss:0.049715885673767876\n",
      "train loss:0.10792112146893304\n",
      "train loss:0.027396653187899803\n",
      "train loss:0.03252010721117616\n",
      "train loss:0.030857837206147428\n",
      "train loss:0.038406197885538784\n",
      "train loss:0.020659214406122294\n",
      "train loss:0.11338615453708849\n",
      "train loss:0.023130546972655916\n",
      "train loss:0.030539014138324477\n",
      "train loss:0.030187914598448638\n",
      "train loss:0.036747650295714786\n",
      "train loss:0.014003388185415628\n",
      "train loss:0.031330764449518\n",
      "train loss:0.09891257815542755\n",
      "train loss:0.03426454177289376\n",
      "train loss:0.0419525179827444\n",
      "train loss:0.009572792905071743\n",
      "train loss:0.06935906187857005\n",
      "train loss:0.0790011799838625\n",
      "train loss:0.0538778932850382\n",
      "train loss:0.01936795489806814\n",
      "train loss:0.048850167168918934\n",
      "train loss:0.019187105348552846\n",
      "train loss:0.10461886477918857\n",
      "train loss:0.07721289011363044\n",
      "train loss:0.011367592204800724\n",
      "train loss:0.04956051280435024\n",
      "train loss:0.022899883282798593\n",
      "train loss:0.0599280021716975\n",
      "train loss:0.0412374497295608\n",
      "train loss:0.0429786244323569\n",
      "train loss:0.16946881094498992\n",
      "train loss:0.021272052794426233\n",
      "train loss:0.019512366886098028\n",
      "train loss:0.005811883748498914\n",
      "train loss:0.04506724086721081\n",
      "train loss:0.04258916787050457\n",
      "train loss:0.016347599611302176\n",
      "train loss:0.033757716714996965\n",
      "train loss:0.08114820671245887\n",
      "train loss:0.04914297579528657\n",
      "train loss:0.019448035976629256\n",
      "train loss:0.028422212182523925\n",
      "train loss:0.040099182589808074\n",
      "train loss:0.09333620453616648\n",
      "train loss:0.02758242927599951\n",
      "train loss:0.045588319663530184\n",
      "train loss:0.0244984394569092\n",
      "train loss:0.10189272921449864\n",
      "train loss:0.009277379627780849\n",
      "train loss:0.09441142170626661\n",
      "train loss:0.06300548061597198\n",
      "train loss:0.016459981791876008\n",
      "train loss:0.02810526127536818\n",
      "train loss:0.022445940221449696\n",
      "train loss:0.21235257417841338\n",
      "train loss:0.040663526369794596\n",
      "train loss:0.035962329101053195\n",
      "train loss:0.09835725622316613\n",
      "train loss:0.007582737847128877\n",
      "train loss:0.02548309856309121\n",
      "train loss:0.0473780598734774\n",
      "train loss:0.07360082867447176\n",
      "train loss:0.1567682449307683\n",
      "train loss:0.049987599118771334\n",
      "train loss:0.05769347725784803\n",
      "train loss:0.015283424932084682\n",
      "train loss:0.006050577358158537\n",
      "train loss:0.06353295670774538\n",
      "train loss:0.056642740548800585\n",
      "train loss:0.054108919278577916\n",
      "train loss:0.06298766220277012\n",
      "train loss:0.04701147452793977\n",
      "train loss:0.019708662254610623\n",
      "train loss:0.024857684245864094\n",
      "train loss:0.015868126494915022\n",
      "train loss:0.028533076157666107\n",
      "train loss:0.015216164890084673\n",
      "train loss:0.06529986547389904\n",
      "train loss:0.046621188867713534\n",
      "train loss:0.07386691456549284\n",
      "train loss:0.02262739078769334\n",
      "train loss:0.08715088595273959\n",
      "train loss:0.039107868471510904\n",
      "train loss:0.0460141752939891\n",
      "train loss:0.008825445048524653\n",
      "train loss:0.031735425430146735\n",
      "train loss:0.02070021104831816\n",
      "train loss:0.01646207898107584\n",
      "train loss:0.008854242573540966\n",
      "train loss:0.004726021046921671\n",
      "train loss:0.05125295016943006\n",
      "train loss:0.015095936308290394\n",
      "train loss:0.01694955223927024\n",
      "train loss:0.03400278644420029\n",
      "train loss:0.07073310672067357\n",
      "train loss:0.026681804406389387\n",
      "train loss:0.052019860936093544\n",
      "train loss:0.017194015663484104\n",
      "train loss:0.060024666976684724\n",
      "train loss:0.014783826839667707\n",
      "train loss:0.037899326389169076\n",
      "train loss:0.028512488776953178\n",
      "train loss:0.026867121693063756\n",
      "train loss:0.07716147278779655\n",
      "train loss:0.010863166551819157\n",
      "train loss:0.029308893498193277\n",
      "train loss:0.06424415771956632\n",
      "train loss:0.11052243399195172\n",
      "train loss:0.04225097108003805\n",
      "train loss:0.01671032476910673\n",
      "train loss:0.06066222490245203\n",
      "train loss:0.02940567494970092\n",
      "train loss:0.007327627953915275\n",
      "train loss:0.005434781637110546\n",
      "train loss:0.023028360571719194\n",
      "train loss:0.02646670404662615\n",
      "train loss:0.022785527405806114\n",
      "train loss:0.025473210534106675\n",
      "train loss:0.03677835190130692\n",
      "train loss:0.07651461014429681\n",
      "train loss:0.0201754112284775\n",
      "train loss:0.024400922612701045\n",
      "train loss:0.0441014543457108\n",
      "train loss:0.028599551862386052\n",
      "train loss:0.030393805434203378\n",
      "train loss:0.061894662956488655\n",
      "train loss:0.06073066415485149\n",
      "train loss:0.0835312716235391\n",
      "train loss:0.008092310459017189\n",
      "train loss:0.022858185232047305\n",
      "train loss:0.021894182937296357\n",
      "train loss:0.0591570285493698\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.012584324981424911\n",
      "train loss:0.1582804716722001\n",
      "train loss:0.10749418981775259\n",
      "train loss:0.027609593538788068\n",
      "train loss:0.06298905994288763\n",
      "train loss:0.02000205144029239\n",
      "train loss:0.027506493323358434\n",
      "train loss:0.03136661800330458\n",
      "train loss:0.052888906870749104\n",
      "train loss:0.07734995296208798\n",
      "train loss:0.05040644356384548\n",
      "train loss:0.005967599883954459\n",
      "train loss:0.025329433284951564\n",
      "train loss:0.09659683009302339\n",
      "train loss:0.06674924332863207\n",
      "train loss:0.12148329670336423\n",
      "train loss:0.027933228848203883\n",
      "train loss:0.02413689554391127\n",
      "train loss:0.05738158757697545\n",
      "train loss:0.01401236161593768\n",
      "train loss:0.015243327453492037\n",
      "train loss:0.013830118378384432\n",
      "train loss:0.08320520453670126\n",
      "train loss:0.018673985759844493\n",
      "train loss:0.03708206252634465\n",
      "train loss:0.016481257294759136\n",
      "train loss:0.03982574917835785\n",
      "train loss:0.037878471054327484\n",
      "train loss:0.054950403322794196\n",
      "train loss:0.010609651318674241\n",
      "train loss:0.03136065195743631\n",
      "train loss:0.07113668895987747\n",
      "train loss:0.07789520747836015\n",
      "train loss:0.06338415937187458\n",
      "train loss:0.03164919587516988\n",
      "train loss:0.047350672528090575\n",
      "train loss:0.025188984245130053\n",
      "train loss:0.03103950085889176\n",
      "train loss:0.04103993785517979\n",
      "train loss:0.05517993664096942\n",
      "train loss:0.05627089049281522\n",
      "train loss:0.024724507662491523\n",
      "train loss:0.03610581843456278\n",
      "train loss:0.03691120907041654\n",
      "train loss:0.027815473040242504\n",
      "train loss:0.01304255346074381\n",
      "train loss:0.027543993011840663\n",
      "train loss:0.01808886473895532\n",
      "train loss:0.05771505958582239\n",
      "train loss:0.02997478464654705\n",
      "train loss:0.022778982811991436\n",
      "train loss:0.01746127867589506\n",
      "train loss:0.011836970046533177\n",
      "train loss:0.04412888191825855\n",
      "train loss:0.012074512516109892\n",
      "train loss:0.009511723895832571\n",
      "train loss:0.04247440983270951\n",
      "train loss:0.03790878511269741\n",
      "train loss:0.06367592148892358\n",
      "train loss:0.0840022456747734\n",
      "train loss:0.04667099685880942\n",
      "train loss:0.022801404828515098\n",
      "train loss:0.10142298252974305\n",
      "train loss:0.11288101184905619\n",
      "train loss:0.03448796806688533\n",
      "train loss:0.043282351223353374\n",
      "train loss:0.03090337197865438\n",
      "train loss:0.06907710409465963\n",
      "train loss:0.01844003897914016\n",
      "train loss:0.020941495487579773\n",
      "train loss:0.05487947919886196\n",
      "train loss:0.03450373771915994\n",
      "train loss:0.02353121808668297\n",
      "train loss:0.01649927574295327\n",
      "train loss:0.06350509623103165\n",
      "epoch:2, train acc:0.9700833333333333, test acc:0.9676 ===\n",
      "train loss:0.02316417049592535\n",
      "train loss:0.0828541126617505\n",
      "train loss:0.02742438515386072\n",
      "train loss:0.009021736786521658\n",
      "train loss:0.06304296147597126\n",
      "train loss:0.10187848393390068\n",
      "train loss:0.010717036313917508\n",
      "train loss:0.015384926703543342\n",
      "train loss:0.011623654239312968\n",
      "train loss:0.08466297170671137\n",
      "train loss:0.029493314656019745\n",
      "train loss:0.003528433323857201\n",
      "train loss:0.04304935443072117\n",
      "train loss:0.07201821081031363\n",
      "train loss:0.014521930530073765\n",
      "train loss:0.035473357415865714\n",
      "train loss:0.09259004902982554\n",
      "train loss:0.05341604750219717\n",
      "train loss:0.11644917969990475\n",
      "train loss:0.03743710615453261\n",
      "train loss:0.03353175215581469\n",
      "train loss:0.08434342802910798\n",
      "train loss:0.01893094904420098\n",
      "train loss:0.03428000311772365\n",
      "train loss:0.009257635723049216\n",
      "train loss:0.028889144769160394\n",
      "train loss:0.10973878911176664\n",
      "train loss:0.040396992230735194\n",
      "train loss:0.07762910708265852\n",
      "train loss:0.027549820441084547\n",
      "train loss:0.06307051113262448\n",
      "train loss:0.0884442648924491\n",
      "train loss:0.02201696319898266\n",
      "train loss:0.05872082949378338\n",
      "train loss:0.017536722106345142\n",
      "train loss:0.027180261785160923\n",
      "train loss:0.011551396222975636\n",
      "train loss:0.019319164835021597\n",
      "train loss:0.06140925267101524\n",
      "train loss:0.04505674920937661\n",
      "train loss:0.006154085699142248\n",
      "train loss:0.06308755575360524\n",
      "train loss:0.03613936485284194\n",
      "train loss:0.07177834985078316\n",
      "train loss:0.0187389742899215\n",
      "train loss:0.01562765655155343\n",
      "train loss:0.03903432623500853\n",
      "train loss:0.039151832508653735\n",
      "train loss:0.03566655834395884\n",
      "train loss:0.008998542155023697\n",
      "train loss:0.0746879952628751\n",
      "train loss:0.015753028090673294\n",
      "train loss:0.03027337528056942\n",
      "train loss:0.07939681263611925\n",
      "train loss:0.024902172588284234\n",
      "train loss:0.014941954231943668\n",
      "train loss:0.035373186435225315\n",
      "train loss:0.056367659257639\n",
      "train loss:0.006759559345657815\n",
      "train loss:0.018888198644022202\n",
      "train loss:0.024532523836154632\n",
      "train loss:0.04398046640628121\n",
      "train loss:0.01634843236187495\n",
      "train loss:0.00883154707320735\n",
      "train loss:0.04588603846895438\n",
      "train loss:0.011023561802433905\n",
      "train loss:0.01413695214315267\n",
      "train loss:0.12791365313427533\n",
      "train loss:0.01967784262709695\n",
      "train loss:0.10063083319651839\n",
      "train loss:0.012638023044536229\n",
      "train loss:0.04068936763272567\n",
      "train loss:0.04565823795254354\n",
      "train loss:0.015426608317587059\n",
      "train loss:0.016195617815835985\n",
      "train loss:0.06665461791045091\n",
      "train loss:0.015417325071039096\n",
      "train loss:0.015125649203319508\n",
      "train loss:0.08290890246189905\n",
      "train loss:0.06608457583230472\n",
      "train loss:0.013363465006021839\n",
      "train loss:0.01038183578013972\n",
      "train loss:0.006034104794131181\n",
      "train loss:0.0120747597179287\n",
      "train loss:0.03374658534704453\n",
      "train loss:0.04268049605343987\n",
      "train loss:0.05191900555705131\n",
      "train loss:0.03453997774650365\n",
      "train loss:0.04307433651466555\n",
      "train loss:0.008216755531239338\n",
      "train loss:0.006889600887635782\n",
      "train loss:0.03666178138194165\n",
      "train loss:0.062224121362434416\n",
      "train loss:0.011153268772379574\n",
      "train loss:0.04611279576008833\n",
      "train loss:0.028216102823447484\n",
      "train loss:0.007138752852305788\n",
      "train loss:0.0024868884167727713\n",
      "train loss:0.003266162100749487\n",
      "train loss:0.00703727132532178\n",
      "train loss:0.061749086283152066\n",
      "train loss:0.021709666362436337\n",
      "train loss:0.007164347827545424\n",
      "train loss:0.1472209099983963\n",
      "train loss:0.0056364061622203475\n",
      "train loss:0.016463706306000733\n",
      "train loss:0.03821219833754946\n",
      "train loss:0.0032152696351239675\n",
      "train loss:0.018360023170182278\n",
      "train loss:0.0665137374863428\n",
      "train loss:0.025817776366773453\n",
      "train loss:0.0497169742755291\n",
      "train loss:0.019536733020161026\n",
      "train loss:0.0033554920303453467\n",
      "train loss:0.0338939878106864\n",
      "train loss:0.068295130213904\n",
      "train loss:0.05667513727217069\n",
      "train loss:0.0176235731369337\n",
      "train loss:0.01935112026695122\n",
      "train loss:0.012680358934854889\n",
      "train loss:0.011138291654956264\n",
      "train loss:0.014983289345365428\n",
      "train loss:0.020580380632130333\n",
      "train loss:0.07825208183318312\n",
      "train loss:0.01274365687436138\n",
      "train loss:0.016757367047709925\n",
      "train loss:0.07267601766489044\n",
      "train loss:0.01819666160235503\n",
      "train loss:0.01068203037621499\n",
      "train loss:0.017899357858421495\n",
      "train loss:0.028729678576116743\n",
      "train loss:0.048287477363954806\n",
      "train loss:0.07521422626347199\n",
      "train loss:0.0900921905792427\n",
      "train loss:0.008735024614599124\n",
      "train loss:0.013110327332379785\n",
      "train loss:0.03530730616618778\n",
      "train loss:0.038567431527531504\n",
      "train loss:0.03715838818335497\n",
      "train loss:0.06500515850813521\n",
      "train loss:0.006022603008045005\n",
      "train loss:0.019150704738992334\n",
      "train loss:0.10299093736510886\n",
      "train loss:0.026191510378988935\n",
      "train loss:0.023002598835973963\n",
      "train loss:0.011195954055429877\n",
      "train loss:0.04805597592464936\n",
      "train loss:0.06949323953185046\n",
      "train loss:0.008307626035614732\n",
      "train loss:0.07728931786401773\n",
      "train loss:0.0727562982718899\n",
      "train loss:0.03920660042965199\n",
      "train loss:0.05321238041733716\n",
      "train loss:0.03778357058797884\n",
      "train loss:0.01646887028556078\n",
      "train loss:0.01928310445626648\n",
      "train loss:0.07059439166277855\n",
      "train loss:0.06458068244710687\n",
      "train loss:0.040703861840536065\n",
      "train loss:0.02651840569029863\n",
      "train loss:0.052325193657136955\n",
      "train loss:0.028456978477843023\n",
      "train loss:0.11958300345784226\n",
      "train loss:0.03712883784165028\n",
      "train loss:0.030024507336687063\n",
      "train loss:0.009024007705108184\n",
      "train loss:0.05744860333476739\n",
      "train loss:0.04072370498214475\n",
      "train loss:0.035644793543632536\n",
      "train loss:0.0443058494545981\n",
      "train loss:0.16858998004782338\n",
      "train loss:0.057105066119397155\n",
      "train loss:0.0434118416554957\n",
      "train loss:0.04498529360567265\n",
      "train loss:0.040392321531866934\n",
      "train loss:0.05023110358790218\n",
      "train loss:0.020219118158456006\n",
      "train loss:0.02958038534048178\n",
      "train loss:0.01985178818214331\n",
      "train loss:0.0193630873385433\n",
      "train loss:0.04684990606818022\n",
      "train loss:0.01307412362715206\n",
      "train loss:0.019733136699000467\n",
      "train loss:0.011356858810897177\n",
      "train loss:0.029572299083418935\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.029991554611987846\n",
      "train loss:0.024303972983159276\n",
      "train loss:0.02734857684646201\n",
      "train loss:0.024153859599369266\n",
      "train loss:0.032212456925871685\n",
      "train loss:0.0016839806905830446\n",
      "train loss:0.0047778175992687095\n",
      "train loss:0.01401868673531026\n",
      "train loss:0.014397199535156825\n",
      "train loss:0.012474973478119959\n",
      "train loss:0.046065623276175575\n",
      "train loss:0.06478345100000302\n",
      "train loss:0.08166572657753367\n",
      "train loss:0.0595557516829687\n",
      "train loss:0.02054707709673802\n",
      "train loss:0.008231296387225123\n",
      "train loss:0.031378180419480546\n",
      "train loss:0.021556124516309755\n",
      "train loss:0.16025313450101936\n",
      "train loss:0.04511141442643814\n",
      "train loss:0.0977877565965299\n",
      "train loss:0.0793930003357132\n",
      "train loss:0.010545441151752311\n",
      "train loss:0.030897332567483823\n",
      "train loss:0.12227026133171295\n",
      "train loss:0.06699677401370135\n",
      "train loss:0.058005128250335034\n",
      "train loss:0.035280102882849024\n",
      "train loss:0.015809073050249735\n",
      "train loss:0.03167043306428396\n",
      "train loss:0.038031925574717344\n",
      "train loss:0.04005891718443248\n",
      "train loss:0.016714273338797613\n",
      "train loss:0.02055909682600145\n",
      "train loss:0.010514600357676993\n",
      "train loss:0.010548146518991499\n",
      "train loss:0.01579789939970108\n",
      "train loss:0.013168147029321781\n",
      "train loss:0.0070453909972555614\n",
      "train loss:0.031233839888977866\n",
      "train loss:0.014642320452582685\n",
      "train loss:0.030626790609869246\n",
      "train loss:0.08861269041821009\n",
      "train loss:0.005497528513102522\n",
      "train loss:0.016603061089802945\n",
      "train loss:0.006734339604920483\n",
      "train loss:0.004820920731547471\n",
      "train loss:0.021127915906124582\n",
      "train loss:0.00597171542422493\n",
      "train loss:0.038620103274269915\n",
      "train loss:0.015051135270189414\n",
      "train loss:0.05945989914459176\n",
      "train loss:0.034882610914648254\n",
      "train loss:0.0013272569837124426\n",
      "train loss:0.05087843018910149\n",
      "train loss:0.1578922899581297\n",
      "train loss:0.023534385800407814\n",
      "train loss:0.018820106262143838\n",
      "train loss:0.025685932727289137\n",
      "train loss:0.026607792663562068\n",
      "train loss:0.019419743917218738\n",
      "train loss:0.02107762370232669\n",
      "train loss:0.02791735034816788\n",
      "train loss:0.02320767663899559\n",
      "train loss:0.013085030176909008\n",
      "train loss:0.03369816777496613\n",
      "train loss:0.13272708523255383\n",
      "train loss:0.02959794812392756\n",
      "train loss:0.026979838006224933\n",
      "train loss:0.012193102693111213\n",
      "train loss:0.0065763896722379036\n",
      "train loss:0.008641451842631032\n",
      "train loss:0.05774424294636111\n",
      "train loss:0.04072486892748538\n",
      "train loss:0.04648696471719546\n",
      "train loss:0.05774487602853769\n",
      "train loss:0.031722943455905976\n",
      "train loss:0.027876193798026767\n",
      "train loss:0.019216752026793234\n",
      "train loss:0.04622108681514572\n",
      "train loss:0.07951822025613117\n",
      "train loss:0.02792364452901234\n",
      "train loss:0.03592449174321249\n",
      "train loss:0.019955938985820345\n",
      "train loss:0.025750802475500892\n",
      "train loss:0.058283469087003235\n",
      "train loss:0.02413217360093458\n",
      "train loss:0.025363167301595678\n",
      "train loss:0.03946968441108287\n",
      "train loss:0.012347172570979286\n",
      "train loss:0.023769833800360785\n",
      "train loss:0.010756623578558206\n",
      "train loss:0.041799260037328326\n",
      "train loss:0.03706983405837297\n",
      "train loss:0.03089036403455542\n",
      "train loss:0.013012719700150657\n",
      "train loss:0.0075085498313651735\n",
      "train loss:0.01902091462678322\n",
      "train loss:0.05091302635245356\n",
      "train loss:0.014943386558051903\n",
      "train loss:0.014542027298840677\n",
      "train loss:0.028652120789225598\n",
      "train loss:0.012193163067090156\n",
      "train loss:0.011496166485170259\n",
      "train loss:0.018783486121772594\n",
      "train loss:0.01763115276189085\n",
      "train loss:0.009241507105106727\n",
      "train loss:0.044510474733591855\n",
      "train loss:0.03428580461276678\n",
      "train loss:0.024886869144156564\n",
      "train loss:0.005307800882000227\n",
      "train loss:0.03851380970636109\n",
      "train loss:0.013288033329491668\n",
      "train loss:0.004634151183734478\n",
      "train loss:0.011740869291202218\n",
      "train loss:0.033371440147448264\n",
      "train loss:0.050816445465706635\n",
      "train loss:0.0281880625743962\n",
      "train loss:0.003751329518993285\n",
      "train loss:0.05091853452078422\n",
      "train loss:0.01661903725739069\n",
      "train loss:0.06004358323538933\n",
      "train loss:0.007447808541099441\n",
      "train loss:0.006477209124366957\n",
      "train loss:0.03677814566660013\n",
      "train loss:0.026755640272588584\n",
      "train loss:0.02907466897397548\n",
      "train loss:0.032745673692147224\n",
      "train loss:0.05166239370047902\n",
      "train loss:0.0025840837220201013\n",
      "train loss:0.016581908785944607\n",
      "train loss:0.023612294921128863\n",
      "train loss:0.009913443426587716\n",
      "train loss:0.03900847896658091\n",
      "train loss:0.014752626155664497\n",
      "train loss:0.022189937280535352\n",
      "train loss:0.03406224063699693\n",
      "train loss:0.005756867063338402\n",
      "train loss:0.01513440499712692\n",
      "train loss:0.06059950387435593\n",
      "train loss:0.04571023684911964\n",
      "train loss:0.021729380297523776\n",
      "train loss:0.0038256585921128076\n",
      "train loss:0.005447476085604868\n",
      "train loss:0.05338375352330995\n",
      "train loss:0.012960680026138729\n",
      "train loss:0.016650150467237602\n",
      "train loss:0.004796458417027429\n",
      "train loss:0.06472130395925565\n",
      "train loss:0.014666761841340975\n",
      "train loss:0.005418695002673752\n",
      "train loss:0.007658884773219064\n",
      "train loss:0.06063585779234603\n",
      "train loss:0.015768371203397594\n",
      "train loss:0.07554271988601066\n",
      "train loss:0.01334227973265168\n",
      "train loss:0.0035558980356644987\n",
      "train loss:0.03670276495541862\n",
      "train loss:0.06533599675539851\n",
      "train loss:0.025301512662683247\n",
      "train loss:0.021275043961531962\n",
      "train loss:0.012380419579985136\n",
      "train loss:0.018007147385614923\n",
      "train loss:0.11113103406621988\n",
      "train loss:0.0454095742062169\n",
      "train loss:0.02014860868989853\n",
      "train loss:0.011288374147389065\n",
      "train loss:0.019871117692156947\n",
      "train loss:0.02401528970584117\n",
      "train loss:0.04571251459280151\n",
      "train loss:0.027587546288347142\n",
      "train loss:0.026841579978251887\n",
      "train loss:0.01800026282467585\n",
      "train loss:0.01715164150490147\n",
      "train loss:0.007305115201617615\n",
      "train loss:0.042333989327338485\n",
      "train loss:0.07056199367770344\n",
      "train loss:0.04306266483985967\n",
      "train loss:0.006090037154220966\n",
      "train loss:0.03271113742009692\n",
      "train loss:0.00937863038557275\n",
      "train loss:0.027793371805075948\n",
      "train loss:0.03765623043334058\n",
      "train loss:0.05172621684525779\n",
      "train loss:0.012218953953955775\n",
      "train loss:0.007706132165150604\n",
      "train loss:0.06665405787778429\n",
      "train loss:0.008155948146909094\n",
      "train loss:0.014033703314365866\n",
      "train loss:0.023178184215209376\n",
      "train loss:0.005654799105937186\n",
      "train loss:0.050223091751284234\n",
      "train loss:0.07343802556666107\n",
      "train loss:0.018158701545817117\n",
      "train loss:0.006302470786932273\n",
      "train loss:0.07808157717287575\n",
      "train loss:0.017320039382826672\n",
      "train loss:0.019561707046295455\n",
      "train loss:0.02340200946184022\n",
      "train loss:0.061340344400578034\n",
      "train loss:0.004744708432112785\n",
      "train loss:0.04332589345258567\n",
      "train loss:0.01894047173572656\n",
      "train loss:0.010286939211610412\n",
      "train loss:0.018707256680292803\n",
      "train loss:0.03884677464088934\n",
      "train loss:0.031193535892144162\n",
      "train loss:0.01178465692741471\n",
      "train loss:0.012904637305760316\n",
      "train loss:0.01106069995614904\n",
      "train loss:0.009913345866076183\n",
      "train loss:0.004942608518430808\n",
      "train loss:0.015176828631482918\n",
      "train loss:0.0056521621688687485\n",
      "train loss:0.11720211693995192\n",
      "train loss:0.05182787923542939\n",
      "train loss:0.020184593940768426\n",
      "train loss:0.03855119186955064\n",
      "train loss:0.0211452423465386\n",
      "train loss:0.009711975842779887\n",
      "train loss:0.009611467322437721\n",
      "train loss:0.006644773874544841\n",
      "train loss:0.026849663006187626\n",
      "train loss:0.02522158962584479\n",
      "train loss:0.010061644939961072\n",
      "train loss:0.00735078924056215\n",
      "train loss:0.006443354228730645\n",
      "train loss:0.01623048587775163\n",
      "train loss:0.031045473519883092\n",
      "train loss:0.013078829721129644\n",
      "train loss:0.024849962872484695\n",
      "train loss:0.01366194383861805\n",
      "train loss:0.005189393409995483\n",
      "train loss:0.014158273638547133\n",
      "train loss:0.016810866416818563\n",
      "train loss:0.006496071918525402\n",
      "train loss:0.0013497863651236166\n",
      "train loss:0.018891010271479613\n",
      "train loss:0.03047856897819406\n",
      "train loss:0.014603584053818697\n",
      "train loss:0.00476284475097605\n",
      "train loss:0.021060906782504968\n",
      "train loss:0.008191047234155426\n",
      "train loss:0.00561522955198147\n",
      "train loss:0.005738298390320335\n",
      "train loss:0.008168964951458833\n",
      "train loss:0.020851172558556893\n",
      "train loss:0.0042929074594579494\n",
      "train loss:0.011131398673331006\n",
      "train loss:0.013507743957773535\n",
      "train loss:0.02498363902957003\n",
      "train loss:0.013648441273033695\n",
      "train loss:0.0013817511700896649\n",
      "train loss:0.007720455201081301\n",
      "train loss:0.009869192890209613\n",
      "train loss:0.0068364973012536865\n",
      "train loss:0.008847494951139977\n",
      "train loss:0.007656652874268706\n",
      "train loss:0.03709474519989166\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.014341647915732923\n",
      "train loss:0.01083879017830295\n",
      "train loss:0.0171813574917614\n",
      "train loss:0.002657302773304827\n",
      "train loss:0.005495685624134876\n",
      "train loss:0.020908714258926753\n",
      "train loss:0.02678330757089146\n",
      "train loss:0.003100310015628714\n",
      "train loss:0.019711034293902165\n",
      "train loss:0.007189082771247861\n",
      "train loss:0.10117613523810429\n",
      "train loss:0.0613386163822597\n",
      "train loss:0.004777689920644329\n",
      "train loss:0.024531416130120408\n",
      "train loss:0.007351811208336643\n",
      "train loss:0.005758885124691361\n",
      "train loss:0.01106294817522329\n",
      "train loss:0.0029479353962975825\n",
      "train loss:0.009160564722751167\n",
      "train loss:0.008916844937342277\n",
      "train loss:0.017389251804789407\n",
      "train loss:0.07658511447363964\n",
      "train loss:0.016040181577029113\n",
      "train loss:0.036652167828529376\n",
      "train loss:0.0010573166542659264\n",
      "train loss:0.002427213623224089\n",
      "train loss:0.005961459175196538\n",
      "train loss:0.01906660000245694\n",
      "train loss:0.011641357737151168\n",
      "train loss:0.037926524250637444\n",
      "train loss:0.024210179611644206\n",
      "train loss:0.058131679650632685\n",
      "train loss:0.028092186090886404\n",
      "train loss:0.06948655749510918\n",
      "train loss:0.08501181609933871\n",
      "train loss:0.005384527494016566\n",
      "train loss:0.027307226700328247\n",
      "train loss:0.0362043847319356\n",
      "train loss:0.014492162102877243\n",
      "train loss:0.019823020441605468\n",
      "train loss:0.06796987022600379\n",
      "train loss:0.01617519468061354\n",
      "train loss:0.024811593793771583\n",
      "train loss:0.018218448436897062\n",
      "train loss:0.028010882484370943\n",
      "train loss:0.012346525560562299\n",
      "train loss:0.019295906989908858\n",
      "train loss:0.010597737865937738\n",
      "train loss:0.007482887825685756\n",
      "train loss:0.0027581100036915313\n",
      "train loss:0.012069101611652821\n",
      "train loss:0.012962107904470006\n",
      "train loss:0.01660608165791098\n",
      "train loss:0.00552243622423023\n",
      "train loss:0.005118951162950912\n",
      "train loss:0.0848405919144036\n",
      "train loss:0.03138901621674042\n",
      "train loss:0.02505306206211269\n",
      "train loss:0.00919971990753701\n",
      "train loss:0.012905539469465845\n",
      "train loss:0.07281937195579208\n",
      "train loss:0.02455413761436075\n",
      "train loss:0.010124037085161115\n",
      "train loss:0.01732262893367027\n",
      "train loss:0.015605124986769848\n",
      "train loss:0.04679281774719055\n",
      "train loss:0.03006285088479439\n",
      "train loss:0.009130181717295443\n",
      "train loss:0.05274627199800715\n",
      "train loss:0.01742046030070129\n",
      "train loss:0.10546162657344273\n",
      "train loss:0.04965021039245785\n",
      "train loss:0.004474515298758904\n",
      "train loss:0.029893073206384\n",
      "train loss:0.02794874127569843\n",
      "train loss:0.009642845821397052\n",
      "train loss:0.011206666963678266\n",
      "train loss:0.017378336249192817\n",
      "train loss:0.009552047629769259\n",
      "train loss:0.03789061432279408\n",
      "train loss:0.019938473184157934\n",
      "train loss:0.005341761978261379\n",
      "train loss:0.02063397517834788\n",
      "train loss:0.05429594752317476\n",
      "train loss:0.05294738262166923\n",
      "train loss:0.02042355303466622\n",
      "train loss:0.0068433386712200795\n",
      "train loss:0.009917023849292762\n",
      "train loss:0.014938346531781554\n",
      "train loss:0.02067708804315563\n",
      "train loss:0.02796156484880645\n",
      "train loss:0.03562130735023869\n",
      "train loss:0.014874168311087255\n",
      "train loss:0.018402203917447374\n",
      "train loss:0.00911717808705075\n",
      "train loss:0.012484657954326796\n",
      "train loss:0.021227000200565657\n",
      "train loss:0.00819954385163381\n",
      "train loss:0.009998534688973316\n",
      "train loss:0.015757167321720618\n",
      "train loss:0.02176960033153745\n",
      "train loss:0.006265550099383042\n",
      "train loss:0.017200260962515486\n",
      "train loss:0.0502852130537514\n",
      "train loss:0.028365123665691713\n",
      "train loss:0.07533108256280491\n",
      "train loss:0.04110598210931687\n",
      "train loss:0.013982253129520008\n",
      "train loss:0.013546970814472294\n",
      "train loss:0.00447816779970776\n",
      "train loss:0.0097794950173814\n",
      "train loss:0.046245393265844544\n",
      "train loss:0.13956096118814482\n",
      "train loss:0.028212571807844235\n",
      "train loss:0.020352246608871853\n",
      "train loss:0.042776773642196\n",
      "train loss:0.02267002125276135\n",
      "train loss:0.0067607115646243195\n",
      "train loss:0.022272598472929745\n",
      "train loss:0.01516749969274731\n",
      "train loss:0.017864565284241338\n",
      "train loss:0.02572347338721486\n",
      "train loss:0.03262017696749585\n",
      "train loss:0.006362574720074993\n",
      "train loss:0.036797313577477134\n",
      "train loss:0.012790626692163434\n",
      "train loss:0.024452462168125345\n",
      "train loss:0.06563603602460516\n",
      "train loss:0.015582678010432567\n",
      "train loss:0.028464299977998195\n",
      "train loss:0.009043825081992382\n",
      "train loss:0.04385661976377201\n",
      "train loss:0.0029916029343346052\n",
      "train loss:0.020319369070018696\n",
      "train loss:0.006312784761061475\n",
      "train loss:0.023464517298566044\n",
      "train loss:0.12987518331574927\n",
      "train loss:0.014158979503160474\n",
      "train loss:0.02659792527170408\n",
      "train loss:0.02162551758544449\n",
      "train loss:0.020282735360129298\n",
      "train loss:0.008714065892963417\n",
      "train loss:0.008247537875946603\n",
      "train loss:0.0694491191420862\n",
      "train loss:0.041027715536786497\n",
      "train loss:0.017588417656520087\n",
      "train loss:0.05079510488366276\n",
      "train loss:0.028362235227018723\n",
      "train loss:0.02267527230124346\n",
      "train loss:0.03360498062147616\n",
      "train loss:0.01194810418664198\n",
      "train loss:0.010308529878612709\n",
      "train loss:0.11768762132408414\n",
      "train loss:0.0893233263901852\n",
      "train loss:0.009875987227326563\n",
      "train loss:0.04896448364387565\n",
      "epoch:3, train acc:0.9873333333333333, test acc:0.983 ===\n",
      "train loss:0.015180494918183199\n",
      "train loss:0.031000922636681322\n",
      "train loss:0.012049132966475052\n",
      "train loss:0.021641494048906588\n",
      "train loss:0.043071922558453817\n",
      "train loss:0.06138468482791311\n",
      "train loss:0.0594451474734299\n",
      "train loss:0.058419149258889747\n",
      "train loss:0.0043412211778774265\n",
      "train loss:0.0035415329164256147\n",
      "train loss:0.0120290666360568\n",
      "train loss:0.004174000108857461\n",
      "train loss:0.016758653834918688\n",
      "train loss:0.020249976070766267\n",
      "train loss:0.0048195840445288095\n",
      "train loss:0.06367274244690103\n",
      "train loss:0.008142146527071698\n",
      "train loss:0.02461595183701745\n",
      "train loss:0.0044809976315349845\n",
      "train loss:0.05307610923495688\n",
      "train loss:0.011323100520836819\n",
      "train loss:0.027439012745854075\n",
      "train loss:0.01601164465951688\n",
      "train loss:0.03824349109753103\n",
      "train loss:0.016006599249642397\n",
      "train loss:0.004592543367499356\n",
      "train loss:0.006099306081765873\n",
      "train loss:0.014640760347943016\n",
      "train loss:0.011492181456158214\n",
      "train loss:0.005202802940382643\n",
      "train loss:0.006441595821999756\n",
      "train loss:0.03105243624086716\n",
      "train loss:0.004342759688650385\n",
      "train loss:0.0313388760406032\n",
      "train loss:0.007682295130813189\n",
      "train loss:0.006743903127780324\n",
      "train loss:0.007940891483100888\n",
      "train loss:0.04222549331568531\n",
      "train loss:0.024868374096366698\n",
      "train loss:0.023476506072981976\n",
      "train loss:0.007828517323270672\n",
      "train loss:0.02091326149025434\n",
      "train loss:0.009263138384585168\n",
      "train loss:0.0025262928207701336\n",
      "train loss:0.019478271308281617\n",
      "train loss:0.014558613640892424\n",
      "train loss:0.010361587774303283\n",
      "train loss:0.0782829316750576\n",
      "train loss:0.0500234879516924\n",
      "train loss:0.04302360395640115\n",
      "train loss:0.020635225750467532\n",
      "train loss:0.0058211263231055625\n",
      "train loss:0.0166658878552585\n",
      "train loss:0.03582550626059245\n",
      "train loss:0.013972568650709939\n",
      "train loss:0.013784445928683619\n",
      "train loss:0.03415706490705761\n",
      "train loss:0.014015053882351226\n",
      "train loss:0.06793428046567435\n",
      "train loss:0.003755688317291187\n",
      "train loss:0.022254022106526047\n",
      "train loss:0.003996126520389933\n",
      "train loss:0.03684819440153116\n",
      "train loss:0.053640082386722104\n",
      "train loss:0.009401482439450486\n",
      "train loss:0.010310600492338643\n",
      "train loss:0.03316538555540136\n",
      "train loss:0.005087224135024608\n",
      "train loss:0.014548684170309866\n",
      "train loss:0.010224069314966713\n",
      "train loss:0.014367221519929448\n",
      "train loss:0.01460905758435344\n",
      "train loss:0.00658835114710428\n",
      "train loss:0.017948296611969505\n",
      "train loss:0.015548986119227375\n",
      "train loss:0.034886845077614484\n",
      "train loss:0.11074825393452663\n",
      "train loss:0.003073933493401261\n",
      "train loss:0.017485410759293135\n",
      "train loss:0.03268099312837278\n",
      "train loss:0.009696978521364216\n",
      "train loss:0.005995582353421557\n",
      "train loss:0.008293168086719725\n",
      "train loss:0.008046077182191502\n",
      "train loss:0.0359137262613285\n",
      "train loss:0.012152933556611335\n",
      "train loss:0.009668652866182221\n",
      "train loss:0.1238714555598228\n",
      "train loss:0.01274941299047291\n",
      "train loss:0.0550028959102718\n",
      "train loss:0.023301652522196674\n",
      "train loss:0.02250949728342298\n",
      "train loss:0.10218006130530224\n",
      "train loss:0.008034128318008265\n",
      "train loss:0.01022618145918451\n",
      "train loss:0.0329377195281686\n",
      "train loss:0.026393372134982596\n",
      "train loss:0.01832719012437332\n",
      "train loss:0.04440940644698599\n",
      "train loss:0.01884564309369718\n",
      "train loss:0.014754772720042999\n",
      "train loss:0.014253162110804532\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.01734744676230532\n",
      "train loss:0.020273208509216403\n",
      "train loss:0.020526063458521294\n",
      "train loss:0.008640435855234002\n",
      "train loss:0.016946620590820718\n",
      "train loss:0.009397108623626966\n",
      "train loss:0.013340728112127314\n",
      "train loss:0.019455446472891488\n",
      "train loss:0.02041464355335854\n",
      "train loss:0.00859391899455845\n",
      "train loss:0.008746771861168212\n",
      "train loss:0.038195793094921014\n",
      "train loss:0.006041071295715759\n",
      "train loss:0.04496761429540189\n",
      "train loss:0.00914609456228122\n",
      "train loss:0.028458952721878827\n",
      "train loss:0.016513806812739906\n",
      "train loss:0.023219028460317332\n",
      "train loss:0.001697425938195626\n",
      "train loss:0.016767554925718325\n",
      "train loss:0.019113233766045158\n",
      "train loss:0.0253649993212349\n",
      "train loss:0.011959155435108015\n",
      "train loss:0.005896068153808248\n",
      "train loss:0.024903775372711285\n",
      "train loss:0.01549247511275742\n",
      "train loss:0.08903143558619699\n",
      "train loss:0.12289931665387034\n",
      "train loss:0.006014286923236673\n",
      "train loss:0.004607867452730735\n",
      "train loss:0.009913164082577043\n",
      "train loss:0.013256997927093091\n",
      "train loss:0.03637456552881049\n",
      "train loss:0.04048792204666985\n",
      "train loss:0.024675969050556044\n",
      "train loss:0.01161159332512737\n",
      "train loss:0.01495228396169823\n",
      "train loss:0.024287365985565402\n",
      "train loss:0.07789626244727488\n",
      "train loss:0.0032123340270493778\n",
      "train loss:0.018936936532743427\n",
      "train loss:0.0032654828997795413\n",
      "train loss:0.01140180885311225\n",
      "train loss:0.0051474533959750815\n",
      "train loss:0.017947920250070356\n",
      "train loss:0.07700310551178108\n",
      "train loss:0.05428753222615689\n",
      "train loss:0.020761611918773494\n",
      "train loss:0.020074756043183428\n",
      "train loss:0.0205887049705714\n",
      "train loss:0.025435099097235875\n",
      "train loss:0.004349087430462127\n",
      "train loss:0.01979739572278351\n",
      "train loss:0.04766962459550404\n",
      "train loss:0.0038600037624652995\n",
      "train loss:0.00658193054831962\n",
      "train loss:0.006371840344912848\n",
      "train loss:0.002398713243088012\n",
      "train loss:0.004632342745636061\n",
      "train loss:0.0026659689809046625\n",
      "train loss:0.018504079353289262\n",
      "train loss:0.012399238472585528\n",
      "train loss:0.014743526962230406\n",
      "train loss:0.0049502583143753425\n",
      "train loss:0.00902308504973274\n",
      "train loss:0.00984377713276783\n",
      "train loss:0.030176947449043253\n",
      "train loss:0.04124539458821437\n",
      "train loss:0.009685092378540712\n",
      "train loss:0.006581322032448076\n",
      "train loss:0.02172323393088468\n",
      "train loss:0.055055709810566025\n",
      "train loss:0.01680013415617072\n",
      "train loss:0.04142180411519023\n",
      "train loss:0.09014211140085887\n",
      "train loss:0.005365547886983017\n",
      "train loss:0.022351139698630597\n",
      "train loss:0.013862639072126285\n",
      "train loss:0.0036085920895597024\n",
      "train loss:0.005286324689082216\n",
      "train loss:0.008476486061755818\n",
      "train loss:0.03781097266143417\n",
      "train loss:0.001973757472365742\n",
      "train loss:0.003646509892529087\n",
      "train loss:0.19654014307876155\n",
      "train loss:0.013794233674584006\n",
      "train loss:0.04601072415128716\n",
      "train loss:0.053451391163626856\n",
      "train loss:0.008882503436463393\n",
      "train loss:0.05546865960338063\n",
      "train loss:0.004236082431279042\n",
      "train loss:0.05261843027273102\n",
      "train loss:0.03718242044276555\n",
      "train loss:0.013417005478745761\n",
      "train loss:0.022290139768066614\n",
      "train loss:0.00980408938386828\n",
      "train loss:0.006546404730655513\n",
      "train loss:0.050716829465770843\n",
      "train loss:0.0029876839351871375\n",
      "train loss:0.007164123923953925\n",
      "train loss:0.035096775814608336\n",
      "train loss:0.023375717314380092\n",
      "train loss:0.005150158992432344\n",
      "train loss:0.012632702908077548\n",
      "train loss:0.016540629611045715\n",
      "train loss:0.014410249154732886\n",
      "train loss:0.014074262562812736\n",
      "train loss:0.014254318746333214\n",
      "train loss:0.023842110017306514\n",
      "train loss:0.01073890250623553\n",
      "train loss:0.016195690326823733\n",
      "train loss:0.01473749629117612\n",
      "train loss:0.037195405439625386\n",
      "train loss:0.00689228326082136\n",
      "train loss:0.03051879542485465\n",
      "train loss:0.004466099320349468\n",
      "train loss:0.006160824212652093\n",
      "train loss:0.01698263516201719\n",
      "train loss:0.04559580928405669\n",
      "train loss:0.005975225527972138\n",
      "train loss:0.02197814827167552\n",
      "train loss:0.022774372110989632\n",
      "train loss:0.0038910103182825282\n",
      "train loss:0.003602608766433359\n",
      "train loss:0.04905645586305587\n",
      "train loss:0.03427065825401769\n",
      "train loss:0.006839648668684075\n",
      "train loss:0.008552358910845704\n",
      "train loss:0.0038813916201462634\n",
      "train loss:0.10198010753553759\n",
      "train loss:0.020820245680235265\n",
      "train loss:0.0008171243401581614\n",
      "train loss:0.011701983248929538\n",
      "train loss:0.0011772160387791208\n",
      "train loss:0.005796467714571041\n",
      "train loss:0.0007429485018340161\n",
      "train loss:0.0028575513550298227\n",
      "train loss:0.0022389611419539423\n",
      "train loss:0.00026331281390598463\n",
      "train loss:0.015438379679229987\n",
      "train loss:0.008691723664924416\n",
      "train loss:0.0077383983339050786\n",
      "train loss:0.006779409869548495\n",
      "train loss:0.002606598223322892\n",
      "train loss:0.005140697943312246\n",
      "train loss:0.007019883251879967\n",
      "train loss:0.0010050282398813989\n",
      "train loss:0.007237052103213348\n",
      "train loss:0.0337650629488647\n",
      "train loss:0.0006851717267459656\n",
      "train loss:0.0022026245716747733\n",
      "train loss:0.010627619885749593\n",
      "train loss:0.007547467178991434\n",
      "train loss:0.01244006652662278\n",
      "train loss:0.003990467620513746\n",
      "train loss:0.011802539780208758\n",
      "train loss:0.01077416596088227\n",
      "train loss:0.0023376916105485495\n",
      "train loss:0.00662693396537737\n",
      "train loss:0.0007067680144576687\n",
      "train loss:0.023088624878614003\n",
      "train loss:0.013178424416076433\n",
      "train loss:0.0011466941555546097\n",
      "train loss:0.0052446786210662314\n",
      "train loss:0.011809638028488601\n",
      "train loss:0.00516926074424052\n",
      "train loss:0.033348116680232034\n",
      "train loss:0.0077101075083268475\n",
      "train loss:0.00657614277617373\n",
      "train loss:0.015717813536444917\n",
      "train loss:0.008154538463729039\n",
      "train loss:0.003216414107119958\n",
      "train loss:0.0013293246125400066\n",
      "train loss:0.0027772791929306278\n",
      "train loss:0.0038239168326776097\n",
      "train loss:0.041626673804260446\n",
      "train loss:0.007989827864077974\n",
      "train loss:0.0012561378238956013\n",
      "train loss:0.06799298593568588\n",
      "train loss:0.01683919922159174\n",
      "train loss:0.021971800971581107\n",
      "train loss:0.03608096958694301\n",
      "train loss:0.006827900368048021\n",
      "train loss:0.002606810268140405\n",
      "train loss:0.014101936072361331\n",
      "train loss:0.009943304264560183\n",
      "train loss:0.03333098533991319\n",
      "train loss:0.006659661890935519\n",
      "train loss:0.003145653520459802\n",
      "train loss:0.04222900948595378\n",
      "train loss:0.030786479972255486\n",
      "train loss:0.004210784443914889\n",
      "train loss:0.0031729545276269096\n",
      "train loss:0.0068195036173028575\n",
      "train loss:0.007249322440107199\n",
      "train loss:0.03672616531176648\n",
      "train loss:0.004767912549453086\n",
      "train loss:0.004446374195856758\n",
      "train loss:0.010651460279665315\n",
      "train loss:0.03272403539309208\n",
      "train loss:0.0030801952202463745\n",
      "train loss:0.010070177756966973\n",
      "train loss:0.00711760905885783\n",
      "train loss:0.0026590439297778513\n",
      "train loss:0.007666955267724024\n",
      "train loss:0.02103643118982187\n",
      "train loss:0.024328916646023165\n",
      "train loss:0.010007331994323895\n",
      "train loss:0.021203853056615905\n",
      "train loss:0.011969486608756212\n",
      "train loss:0.005714540346202525\n",
      "train loss:0.0022902258521201196\n",
      "train loss:0.17003881509732208\n",
      "train loss:0.00999798542995906\n",
      "train loss:0.012586916008886078\n",
      "train loss:0.0045590442405733025\n",
      "train loss:0.007323494774626995\n",
      "train loss:0.004946817259450076\n",
      "train loss:0.013587923949348388\n",
      "train loss:0.020286141805333238\n",
      "train loss:0.010100306984351794\n",
      "train loss:0.01972831126548355\n",
      "train loss:0.003401902542899973\n",
      "train loss:0.028654996228199624\n",
      "train loss:0.013827796367973853\n",
      "train loss:0.0041222742782842485\n",
      "train loss:0.007406555046435991\n",
      "train loss:0.0474384595107871\n",
      "train loss:0.0030571432848473866\n",
      "train loss:0.012768287917115574\n",
      "train loss:0.004834411549274248\n",
      "train loss:0.0028783002305248083\n",
      "train loss:0.006518173548738404\n",
      "train loss:0.017747090763350723\n",
      "train loss:0.007103771410596582\n",
      "train loss:0.008761786323224732\n",
      "train loss:0.037441672959359275\n",
      "train loss:0.003038818383355824\n",
      "train loss:0.027079868409813575\n",
      "train loss:0.007415637260344729\n",
      "train loss:0.0029790766962135866\n",
      "train loss:0.02083872088750506\n",
      "train loss:0.03890788288620762\n",
      "train loss:0.020565180810448464\n",
      "train loss:0.012459798722650411\n",
      "train loss:0.008774538161591435\n",
      "train loss:0.009748517534695637\n",
      "train loss:0.023156894324172977\n",
      "train loss:0.002158197136602627\n",
      "train loss:0.009286681433382113\n",
      "train loss:0.004267765704555111\n",
      "train loss:0.004364269758323503\n",
      "train loss:0.005562703794195948\n",
      "train loss:0.002085599239699683\n",
      "train loss:0.006795010387316932\n",
      "train loss:0.031702301582988326\n",
      "train loss:0.0006184574762364436\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss:0.030160575608379633\n",
      "train loss:0.0042117206884373344\n",
      "train loss:0.0002289263338345795\n",
      "train loss:0.007929870595452455\n",
      "train loss:0.062253551798914335\n",
      "train loss:0.03440914672799412\n",
      "train loss:0.04722847772839115\n",
      "train loss:0.012907677183519888\n",
      "train loss:0.012567308895381865\n",
      "train loss:0.0038850910509181085\n",
      "train loss:0.009869428433902509\n",
      "train loss:0.004112183277594726\n",
      "train loss:0.014193943384306056\n",
      "train loss:0.01381942980452584\n",
      "train loss:0.05346960100385534\n",
      "train loss:0.005385921686751377\n",
      "train loss:0.008351491321974882\n",
      "train loss:0.02342767609762606\n",
      "train loss:0.12055410972415989\n",
      "train loss:0.007174212752262075\n",
      "train loss:0.015126349469912702\n",
      "train loss:0.006164783583047242\n",
      "train loss:0.010820702299306451\n",
      "train loss:0.01210884910107303\n",
      "train loss:0.022687554214688245\n",
      "train loss:0.0016381133073314071\n",
      "train loss:0.011182378795367829\n",
      "train loss:0.010685803381059766\n",
      "train loss:0.03268453728467306\n",
      "train loss:0.03621267434681208\n",
      "train loss:0.028472708056445126\n",
      "train loss:0.01949589185283799\n",
      "train loss:0.028777979767955084\n",
      "train loss:0.00790085216031842\n",
      "train loss:0.1618669296332394\n",
      "train loss:0.007055699632256052\n",
      "train loss:0.04093558175685733\n",
      "train loss:0.018594704352652026\n",
      "train loss:0.06228249562551282\n",
      "train loss:0.02826916657302441\n",
      "train loss:0.06524677068860979\n",
      "train loss:0.014882395457181037\n",
      "train loss:0.04748323759839525\n",
      "train loss:0.022812256490942907\n",
      "train loss:0.01915699015339962\n",
      "train loss:0.01783928039106484\n",
      "train loss:0.08520174830629244\n",
      "train loss:0.028036297364855356\n",
      "train loss:0.13847516225181777\n",
      "train loss:0.03562109105463826\n",
      "train loss:0.030650464556634255\n",
      "train loss:0.005773507361921928\n",
      "train loss:0.03781877294030207\n",
      "train loss:0.019070339465540476\n",
      "train loss:0.0187185828843366\n",
      "train loss:0.024414040788597\n",
      "train loss:0.01739277799645797\n",
      "train loss:0.02113696288574988\n",
      "train loss:0.011353002414527563\n",
      "train loss:0.036107079995802836\n",
      "train loss:0.008436057406965083\n",
      "train loss:0.017514402170017283\n",
      "train loss:0.008302557937103\n",
      "train loss:0.04927583951150848\n",
      "train loss:0.003392369589811013\n",
      "train loss:0.005226105198803915\n",
      "train loss:0.042195580170244475\n",
      "train loss:0.040809465250632775\n",
      "train loss:0.029684217601088032\n",
      "train loss:0.0019798057202525905\n",
      "train loss:0.004524950788605446\n",
      "train loss:0.008718525915886156\n",
      "train loss:0.018363293293503523\n",
      "train loss:0.009399672366631355\n",
      "train loss:0.00274270733505178\n",
      "train loss:0.01063040152484601\n",
      "train loss:0.020202992570569598\n",
      "train loss:0.026332392595468566\n",
      "train loss:0.04664540296094361\n",
      "train loss:0.003470056740458205\n",
      "train loss:0.014811111203378935\n",
      "train loss:0.0033721631969308523\n",
      "train loss:0.028385104656303152\n",
      "train loss:0.0015249118252641603\n",
      "train loss:0.002784130766487479\n",
      "train loss:0.021942090152660066\n",
      "train loss:0.004391569409375259\n",
      "train loss:0.022787270625077117\n",
      "train loss:0.03945653871198775\n",
      "train loss:0.012039116912487378\n",
      "train loss:0.005025887870367376\n",
      "train loss:0.017627836922626827\n",
      "train loss:0.011602760786087008\n",
      "train loss:0.00191086495342357\n",
      "train loss:0.04149708066881705\n",
      "train loss:0.004565775867229529\n",
      "train loss:0.007130119574944661\n",
      "train loss:0.015638822946402536\n",
      "train loss:0.002088569628292606\n",
      "train loss:0.004480610699414511\n",
      "train loss:0.01121695305121042\n",
      "train loss:0.045419513354637384\n",
      "train loss:0.0007439348001915857\n",
      "train loss:0.0023208001728972716\n",
      "train loss:0.004589334889230587\n",
      "train loss:0.003833678475919585\n",
      "train loss:0.05893001042298301\n",
      "train loss:0.009045710962512084\n",
      "train loss:0.07071074902400148\n",
      "train loss:0.003532387369241346\n",
      "train loss:0.0030575189073695796\n",
      "train loss:0.010353153621472877\n",
      "train loss:0.010812407498617789\n",
      "train loss:0.005365639283044062\n",
      "train loss:0.004332021491990387\n",
      "train loss:0.003757532234113719\n",
      "train loss:0.025532946061828744\n",
      "train loss:0.0016598073336038827\n",
      "train loss:0.0035545278708359763\n",
      "train loss:0.0029303330988274252\n",
      "train loss:0.017561026556371734\n",
      "train loss:0.051470697584192725\n",
      "train loss:0.018784632965743776\n",
      "train loss:0.0021149071719762783\n",
      "train loss:0.011731738738412947\n",
      "train loss:0.007860707127583962\n",
      "train loss:0.007860587428592218\n",
      "train loss:0.0044093542620586325\n",
      "train loss:0.001963774504334101\n",
      "train loss:0.02808548609161201\n",
      "train loss:0.06126574653447192\n",
      "train loss:0.004597816179636299\n",
      "train loss:0.028017858075933905\n",
      "train loss:0.010977833265109236\n",
      "train loss:0.12835551803592196\n",
      "train loss:0.05680850962982681\n",
      "train loss:0.00852841507589294\n",
      "train loss:0.014302715566451797\n",
      "train loss:0.03376144824208972\n",
      "train loss:0.004675484552292623\n",
      "train loss:0.003148140435469963\n",
      "train loss:0.03416248057890772\n",
      "train loss:0.012152133549809237\n",
      "train loss:0.04640105287821125\n",
      "train loss:0.018858771211570403\n",
      "train loss:0.027842653126216678\n",
      "train loss:0.007930467377553607\n",
      "train loss:0.010403939350073377\n",
      "train loss:0.010836762495947892\n",
      "train loss:0.01925982305395092\n",
      "train loss:0.0040697728769852224\n",
      "train loss:0.033143693776525346\n",
      "train loss:0.02084678685056612\n",
      "train loss:0.007231438158995285\n",
      "train loss:0.0019099956014920888\n",
      "train loss:0.004390951163693288\n",
      "train loss:0.009639483153394666\n",
      "train loss:0.05239252488023574\n",
      "train loss:0.017619740618483225\n",
      "train loss:0.01059086998158577\n",
      "train loss:0.006707675486889036\n",
      "train loss:0.0097721784140062\n",
      "train loss:0.015229016019413545\n",
      "train loss:0.00857401879659269\n",
      "train loss:0.012564941085383452\n",
      "train loss:0.004697016552662725\n",
      "train loss:0.02986771252175623\n",
      "train loss:0.0032361311094474877\n",
      "train loss:0.005980309172890029\n",
      "train loss:0.007027568331144438\n",
      "train loss:0.01869863385065833\n",
      "train loss:0.03797787378300797\n",
      "train loss:0.03634194551548995\n",
      "train loss:0.007808821096961962\n",
      "train loss:0.022829629078921495\n",
      "train loss:0.0018408116389583952\n",
      "train loss:0.003873090321852439\n",
      "train loss:0.0041208422435291955\n",
      "train loss:0.004817300511356392\n",
      "train loss:0.00537820636110109\n",
      "train loss:0.0055978183688490165\n",
      "train loss:0.04321012489517126\n",
      "train loss:0.02858428960667257\n",
      "train loss:0.007522127388799637\n",
      "train loss:0.0038907546700415506\n",
      "train loss:0.017238716856439077\n",
      "train loss:0.002309055336890509\n",
      "train loss:0.010503023016250255\n",
      "train loss:0.002186055142782488\n",
      "train loss:0.01349417344539448\n",
      "train loss:0.026286204252372363\n",
      "train loss:0.011785097460068334\n",
      "train loss:0.013159346402342119\n",
      "train loss:0.002043618928756169\n",
      "train loss:0.01267709586617416\n",
      "train loss:0.017483869062266958\n",
      "train loss:0.013042523608946708\n",
      "train loss:0.0055387381564218265\n",
      "train loss:0.008225047332477278\n",
      "train loss:0.0004463985091346336\n",
      "train loss:0.009908470797384846\n",
      "train loss:0.018189445115157864\n",
      "train loss:0.0007236433562982831\n",
      "train loss:0.0034032052157406366\n",
      "train loss:0.008210398427001933\n",
      "train loss:0.004469189572048039\n",
      "train loss:0.014588451822252902\n",
      "train loss:0.003723629319160985\n",
      "train loss:0.01589238642659213\n",
      "train loss:0.0013394967851238972\n",
      "train loss:0.0005971910832488053\n",
      "train loss:0.07130425032567789\n",
      "train loss:0.006223856219411675\n",
      "train loss:0.029700073902784393\n",
      "train loss:0.007679002957665765\n",
      "train loss:0.023424122248048587\n",
      "train loss:0.020502957862487233\n",
      "train loss:0.0020673159730543685\n",
      "train loss:0.0031208791673752057\n",
      "train loss:0.003090804123379736\n",
      "train loss:0.0002665721301985853\n",
      "train loss:0.003608909055898773\n",
      "train loss:0.00455373036708543\n",
      "train loss:0.020128377037679782\n",
      "train loss:0.03707473420974936\n",
      "train loss:0.02941009094674139\n",
      "train loss:0.004976760734286963\n",
      "train loss:0.004907736166116386\n",
      "train loss:0.013430006497789275\n",
      "train loss:0.021458869711350515\n",
      "train loss:0.01533837282902194\n",
      "train loss:0.00825847372105527\n",
      "train loss:0.16912438269043115\n",
      "train loss:0.0020825103239451555\n",
      "train loss:0.06026679916791071\n",
      "train loss:0.03293033376813302\n",
      "train loss:0.007050230848103002\n",
      "train loss:0.013486598253567017\n",
      "train loss:0.07278588898476351\n",
      "train loss:0.02045579526169148\n",
      "train loss:0.0026402041227547505\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[35], line 14\u001b[0m\n\u001b[0;32m      9\u001b[0m net \u001b[38;5;241m=\u001b[39m ConvNet(input_dim\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m28\u001b[39m,\u001b[38;5;241m28\u001b[39m))\n\u001b[0;32m     11\u001b[0m trainer \u001b[38;5;241m=\u001b[39m Trainer(net, train_scaled, train_y, test_scaled, test_y, epochs\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m10\u001b[39m, mini_batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m100\u001b[39m, optimizer\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAdam\u001b[39m\u001b[38;5;124m\"\u001b[39m, \n\u001b[0;32m     12\u001b[0m                   optimizer_param\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m\"\u001b[39m:\u001b[38;5;241m0.01\u001b[39m})\n\u001b[1;32m---> 14\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[23], line 59\u001b[0m, in \u001b[0;36mTrainer.train\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtrain\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m     58\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_iter):\n\u001b[1;32m---> 59\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     60\u001b[0m     test_acc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork\u001b[38;5;241m.\u001b[39maccuracy(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mx_test, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_test)\n\u001b[0;32m     61\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbose:\n",
      "Cell \u001b[1;32mIn[23], line 51\u001b[0m, in \u001b[0;36mTrainer.train_step\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     49\u001b[0m     x_train_sample, t_train_sample \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mx_train[:t], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_train[:t]\n\u001b[0;32m     50\u001b[0m     x_test_sample, t_test_sample \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mx_test[:t], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_test[:t]\n\u001b[1;32m---> 51\u001b[0m train_acc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maccuracy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_train_sample\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mt_train_sample\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     52\u001b[0m test_acc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork\u001b[38;5;241m.\u001b[39maccuracy(x_test_sample, t_test_sample)\n\u001b[0;32m     53\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrain_acc_list\u001b[38;5;241m.\u001b[39mappend(train_acc)\n",
      "Cell \u001b[1;32mIn[28], line 51\u001b[0m, in \u001b[0;36mConvNet.accuracy\u001b[1;34m(self, x, t, batch_size)\u001b[0m\n\u001b[0;32m     49\u001b[0m tx \u001b[38;5;241m=\u001b[39m x[i\u001b[38;5;241m*\u001b[39mbatch_size:(i\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m*\u001b[39mbatch_size]\n\u001b[0;32m     50\u001b[0m tt \u001b[38;5;241m=\u001b[39m t[i\u001b[38;5;241m*\u001b[39mbatch_size:(i\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m*\u001b[39mbatch_size]\n\u001b[1;32m---> 51\u001b[0m y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     52\u001b[0m y \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margmax(y, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m     53\u001b[0m acc \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39msum(y \u001b[38;5;241m==\u001b[39m tt)\n",
      "Cell \u001b[1;32mIn[28], line 38\u001b[0m, in \u001b[0;36mConvNet.predict\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpredict\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers\u001b[38;5;241m.\u001b[39mvalues():\n\u001b[1;32m---> 38\u001b[0m         x \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m x\n",
      "Cell \u001b[1;32mIn[18], line 21\u001b[0m, in \u001b[0;36mConvolution.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     19\u001b[0m col \u001b[38;5;241m=\u001b[39m im2col(x, FH, FW, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpad)\n\u001b[0;32m     20\u001b[0m col_W \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mW\u001b[38;5;241m.\u001b[39mreshape(FN, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mT\n\u001b[1;32m---> 21\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcol_W\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mb\u001b[49m\n\u001b[0;32m     22\u001b[0m out \u001b[38;5;241m=\u001b[39m out\u001b[38;5;241m.\u001b[39mreshape(N, out_h, out_w, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m3\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mx \u001b[38;5;241m=\u001b[39m x\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from collections import OrderedDict # 순서가 있는 딕셔너리\n",
    "\n",
    "(train_x, train_y), (test_x, test_y) = keras.datasets.mnist.load_data()\n",
    "\n",
    "train_scaled = train_x.reshape(-1,1,28,28)/255\n",
    "test_scaled = test_x.reshape(-1,1,28,28)/255\n",
    "\n",
    "net = ConvNet(input_dim=(1,28,28))\n",
    "\n",
    "trainer = Trainer(net, train_scaled, train_y, test_scaled, test_y, epochs= 10, mini_batch_size = 100, optimizer= \"Adam\", \n",
    "                  optimizer_param={\"lr\":0.01})\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "57426294",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cherry', 'banana', 'apple', 'date']\n"
     ]
    }
   ],
   "source": [
    "# 원본 리스트\n",
    "my_list = ['apple', 'banana', 'cherry', 'date']\n",
    "# 특정 인덱스 기준으로 정렬 (예: 인덱스 2의 값 기준)\n",
    "index_to_sort = 2\n",
    "sorted_list = sorted(my_list, key=lambda x: x[index_to_sort])\n",
    "print(sorted_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4098496b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}