{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Discussion of the GP marginal likelihood upper bound\n", "--\n", "\n", "*Mark van der Wilk, August 2017* \n", "*Small edits by James Hensman 2017*\n", "\n", "See [gp_upper](https://github.com/markvdw/gp_upper) for code to tighten the upper bound through optimisation, and a more comprehensive discussion." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import matplotlib\n", "%matplotlib inline\n", "matplotlib.rcParams['figure.figsize'] = (12, 6)\n", "plt = matplotlib.pyplot\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import gpflow\n", "\n", "from FITCvsVFE import getTrainingTestData\n", "from __future__ import print_function" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "X, Y, Xt, Yt = getTrainingTestData()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def plot_model(m, name=\"\"):\n", " pX = np.linspace(-3, 9, 100)[:, None]\n", " pY, pYv = m.predict_y(pX)\n", " plt.plot(X, Y, 'x')\n", " plt.plot(pX, pY)\n", " try:\n", <<<<<<< HEAD " plt.plot(m.feature.value, m.feature.Z.value * 0, 'o')\n", ======= " plt.plot(m.feature.Z.value, m.feature.Z.value * 0, 'o')\n", >>>>>>> master " except AttributeError:\n", " pass\n", " two_sigma = (2.0 * pYv ** 0.5)[:, 0]\n", " plt.fill_between(pX[:, 0], pY[:, 0] - two_sigma, pY[:, 0] + two_sigma, alpha=0.15)\n", " lml = m.compute_log_likelihood()\n", " plt.title(\"%s (lml = %f)\" % (name, lml))\n", " return lml" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Full model" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 23.966658\n", " Number of iterations: 13\n", " Number of functions evaluations: 18\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsUAAAF1CAYAAAAA6ZfwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcXFWZP/7PubVXdfXe2buTkJANsjQEElZN2GQJAgoE\nFBVnRHRcvo4zjo6KioM/dVxHBXRUUBCiiGCIOIDsWwiEJoGsJCHpLJ10eq+97nJ+f1R1U73Wdmv/\nvF+vfvVSdU+drqSrnvvc5zxHSClBRERERFTJlEJPgIiIiIio0BgUExEREVHFY1BMRERERBWPQTER\nERERVTwGxURERERU8RgUExEREVHFY1BMRGVHCHG/EOKK+NcfE0K8kOE47xVCHDJ3djSSEMIhhNgp\nhGgq9FyIqHIxKCaisiKEWAJgKYC/FnouqRJC/EAI8bYQwhcPDj+ScFujEOJFIUS3EKJfCPGyEOKs\nCcaaLoT4qxCiRwhxSAhx84jbLUKI/xJCHIk/XpsQojbh9hOEEBvit3UJIb4/4vi1QogdQoiAEGKv\nEOKc+M9nCSGkEMKf8PH1hOMcQog7hRDH4nN7RAgxHQCklBEAvwXw5WyfSyKiTDEoJqJy80kAf5Cl\ntTNRAMAaADUAPgrgp0KIM+O3+QH8M4DJAGoBfA/AI0II6zhj3Qvgnfj9LwXwHSHEqoTbvwXgTABn\nAKgGcAOAMAAIIewAngDwFIApAGbEx0P89gvij38jAC+AcwHsG/H4tVLKqvjHtxN+/vn4Yy4BMA1A\nL4CfJdx+H4CPCiEc4/xeREQ5xaCYiMrNxQCeHe/GeDbz0wmZ2W8LIeYIIV4SQgwIIf4UDw7zRkr5\nDSnlTimlIaV8BcDziAWQkFKGpZQ7pJQaAAFAB1AHoH6M360KwHsBfEdKqUoptwD4M4CPx2+vA/D/\nAHxCSnlAxrwlpQzHh/gYgCNSyh9JKQPxx96a8BDfAnCrlHJjfK6HpZSHU/w1ZwN4TEp5LP54fwRw\nUsJzcAixQHlliuMREZmKQTERlQ0hhAex4GtXkrteBOBUxAKwLwH4FYAPA2gGcDKA61J8vK1CiL5x\nPm7P8HdwATgNwLaRj4VYRnc9gF9LKTvHOnycn50c/3oxAA3AB4UQR4UQu4UQ/5Jw35UA9gsh/h4v\nnXhGCLE4/vgWAMsBNAkh9sRLM34en2+iA/Hb7hJCNCb8/DcAzhJCTBNCuAF8CMDfRxy7A7HSFyKi\nvGNQTETlZLA21pfkft+XUg5IKbcBeAvA41LKfVLKfsQCtdZUHkxKuURKWTvOx6cz/B3uBLAFwGMj\nHwuxcofrAYy5cFBK6QPwIoCvCyGcQohTAHwAgDt+lxmIlWjMQ+zk4YMAvhkvixi8fS2A/0GsxOFv\nAP4az5xPBmCLH3MOgGWIPU9fix/bhVgwPxOxEw4vgD8kTO9tAAcBHAYwAGAhgFtH/Ao+vPtvSESU\nVwyKiaic9MU/e5Pc71jC16Exvq8yc1KJ4ovNBhei/eeI2/4bsazuNWPVRMfLGe4H8GUhxHgZ1Q8h\nFvAeBHAHYjXBgx00QvHPt0opQ/HSiHUALkm4/QUp5d+llFEAPwDQgFgAO3jsz6SUHVLKLgA/GjxW\nSumXUr4mpdSklMcAfAbAhUKIwX+LXwBwxsfzAPgLRmeKvXj335CIKK8YFBNR2ZBSBgDsRSwTmnNC\niG0jui0kftw5zhxvTliI9p2Esb6FWD30hVLKgSQPbQNwwjjjH5BSXialbJJSrgDQCGBT/ObB+uDE\ngDvx660jvk8ctxex4Hq8Y0cdEv88+D6zDMBdUsqeeLeJnwE4fUSJxULEsuRERHnHoJiIys2jAN6T\njweSUp6UEOCO/Lg5+QgxQoivIFYWcb6UsnvEbSuFEGcLIexCCJcQ4j8QK2V4ZZyxFgohvPH7fxjA\nhYhldCGl3IvYIr6vxlukLUSsXGJD/PB7AawUQpwfryH+f4iVReyI334XgM8KISbFF+19YfBYIcQK\nIcR8IYQihGhArATjmXhJCgC8CuAjQogaIYQNwKcRW9TXFT9+OmKLBzem+rwREZmJQTERlZtfAfiQ\nEGKsRWfF6jsAWgDsGaO0woFY6UE3YvW4lwC4VEp5BACEEB8SQiQuyrsIsTZpvQBuBvA+KeXxhNuv\nQ6zutxuxmuGvSymfBAAp5S7EFhzeGT/+/QAuj5dSAMC3EQtudyMWKLcBuC1+2wkA/g+xuuC3AEQw\nfMHivyG2UPBtAMfjv8eVCbdfD+B38SwyEVHeidJq5UlElJwQ4j4Af5JSPlzouVBy8d7EWwCcO05X\nDSKinGNQTEREREQVj+UTRERERFTxGBQTERERUcVjUExEREREFY9BMRERERFVPGshHrSxsVHOmjWr\nEA9NRERERBVk8+bNXVLKpmT3K0hQPGvWLLz22muFeGgiIiIiqiBCiAOp3I/lE0RERERU8RgUExER\nEVHFY1BMRERERBWPQTERERERVTwGxURERERU8RgUExEREVHFY1BMRERERBWPQTERERERVTwGxURE\nRERU8RgUExEREVHFY1BMRERERBWPQTERERERVTxrtgMIIZoB/B7AZAASwK+klD/NdlwiIiKqPIYh\nEdZ06IaEBCAlIKWMfQZgtQhYFQGrosBmERBCFHrKVCayDooBaAC+KKV8XQjhBbBZCPGElHK7CWMT\nERFRmZNSIhDVEYhoCEZ1SClTPtaiCFgtsQDZYbHAblVgtyqwKAyWKT1ZB8VSyg4AHfGvfUKIHQCm\nA2BQTBkbzAoAscyAIsBsABFRmQlFdfgiKoIRHUYagXAi3ZDQDR0RFfBDG/q5RRGwWxXYLLEPR/xr\nBss0HjMyxUOEELMAtAJ4xcxxqbRJKaEbEpoR+6xLCV2PfzaGf0z0omizKLBaxNALnN3CbAARUSmS\nUqLLH4UvrObsMXRDIhTVEYI+7OcW5d33EZsllmWOlWPEvqbKZVpQLISoAvAggP8npRwY4/abANwE\nAC0tLWY9LOWZEQ9cDQkY8WzuYHBrxAPewc+Jwa4ZVN2AqmPYC5wQAl6nFdVOG+xWvpgRERU7w5A4\n5gsjFNWT3zkHBjPLYXX04wsRC44tiR9CQEn4WghAEQLK4GcmZsqGSKduZ9xBhLAB2ADgMSnlj5Ld\nf/ny5fK1117L+nHT5Y9oY/4RlIvx/iljSxWGvhn1ZWzxgkz4OmFRQ/w2GQ+Ci5nHYUWNywanzVLo\nqRAR0RhU3cDR/jBU3Sj0VEwl4kGyQCxoBjAULAtg6GeJtw/eNvyLxNuSB9ulVFVY77YX7ARCCLFZ\nSrk82f3M6D4hAPwGwI5UAuJCCkX1nF6qocIKRDQEIhocNgvq3Xa47AyOiYiKRVjVcWwgbNrVw2Ii\npYQugWGZp/LNwWWk1mWDkkKgX0hmXG8+C8ANAFYLId6If1xiwrhEGYmoOjr6QxjgCRARUVHwRzR0\n9JdnQEzlw4zuEy9gzMQ/UWF1+SLQdYk6j73QUyEiqljBqIbOgXChp0GUFFcmUVnrDUZx3BdJq+cl\nERGZwzAkunzRQk+DKCUMiqns+cIqjg1EYPCyHRFRXnUHotCM8lpUR+WLQTFVhGBUQ0eZLvAgIipG\nXNxOpYZBMVWMiKrj6ECYpRRERDlmGBJd/kihp0GUFgbFVFEiqo6+IDMXRES51BOMll0vYip/DIqp\n4vSF1LLexIWIqJDCqo6BEJMPVHoYFFPFkVKyIwURUQ4Mvr4SlSIGxVSRVN1AT4BtgoiIzNQTYNkE\nlS4GxVSx+kMqQlGWURARmSGi6ehn2QSVMAbFVNG6/OxfTERkBgbEVOoYFFNFU3UDXQHWvxERZUPT\nDQQivPJGpY1BMVU8f1hDIKIVehpERCXLF9a4eJlKHoNiIsQWh/AFnYgofVJKDHDnOioDDIqJECuj\n8DNbTESUNn9Eg861GVQGGBQTxXGnOyKi9HGBHZULBsVEccwWExGlJxTVEdXYl5jKA4NiogR9QW7o\nQUSUKtYSUzlhUEyUIKoZCEaZLSYiSkbVDXbuobLCoJhohF7WFhMRJcVaYio3DIqJRoioOrd/JiKa\ngGFI+MPMElN5YVBMNIa+EGuLiYjG4wtrMNjbncoMg2KiMYSiOsIqs8VERGPhAjsqRwyKicbBvsVE\nRKMFoxpUnW3YqPwwKCYaRzCqIaIxW0xElIj93KlcMSgmmkA/s8VEREOklAhGmCyg8sSgmGgCgagO\n3eBiEiIiAAipOhfYUdliUEw0ASnZdoiIaBBLJ6icMSgmSoKrrImIWDpB5c9a6AkQFTtVNxBWdTht\nlkJPhYjiIpqOiBb724QEmrwOCCEKPa2yxtIJKncMiolS4AtrDIqJCqwvGEVI1RFRjdHBmQAmeZ2F\nmViFYOkElTuWTxClIBDRIJkhISqY/pCKnkAUoejY2Up/WENPgDtR5oqUEqEoSyeovDEoJkqBISUC\nfEMgKoiIpqcU8PYFo1wDkCNh1WAnHip7DIqJUuTjmy1R3hmGROdAJOUrNd3+KIJRXuY3G0snqBIw\nKCZKUSiqQ+PWpkR51RWIpLWlsJSxIJq7UZqLJxpUCRgUE6XBx57FRHkzEFYz6hNuSIlj/ekF0zS+\nEDcxogrBoJgoDbyESJQfUc1Atz/zhXOaYaDLHzFxRpWLr3tUKRgUE6VB1Q2uwCbKMSkljg2Es+74\nEorqiGrMFmeLpRNUKRgUE6XJF+GCO6Jc6vJHTSt94ALZ7LB0gioJg2KiNAUiOgy+SRDlRETTTQ1k\n/ewxnpUAs8RUQRgUE6VJSgk/3yiIcqI/aG5mVzcka2KzEIywXIwqB4NiogxksiKeiCYW1YycBLAD\n/HvNSFjVoRmsyabKwaCYKANhVWe7JyKT9YVys01zRNXZtzgDAWbYqcIwKCbKEN8wiMyj6gYCObxU\nPxDi32u6guy0QxWGQTFRhlinSGSevqCa0wVxgYjGBbJp0HSDV8Oo4jAoJspQVOObBpEZND03tcSJ\nDCnh44lsyoIqs8RUeRgUE2WBJRRE2esPZZ8lXrepHW3tvcN+1tbei3Wb2oe+HwixZ3GqwiydoArE\noJgoCyyhIMqObkhTukPMn+LFrRt2DAXGbe29uHXDDsyf4h26j6obCDMDmpIQnyeqQNZCT4ColEU1\nA1HNgN3K80uiTJiRJQaA1pY63HLZQty6YQcuXzoV67d04JbLFqK1pW7Y/QZCKpw2S9aPV87CKnex\no8rEoJgoS8GoBrvVXuhpUJGLaDq6/FHYFAGH1QKHTYHDqkAIUeipFYxhSFNLGlpb6nD50qm4Z2M7\nbljZMiogBoBAfNtii1K5z3syIZZOUIViUEyUJX9EQ62bQTGNLxTVcWwgDENKRPBu2Y0QAjaLgMdu\nRZ2n8v4PDYRVGCZ2nGhr78X6LR24YWUL1m/pwLLm2lGBsZQSvrDKv9kJcJEdVSpe8yXK0mAJBdFY\n/BENR+MB8UhSSkQ1A73BKPqCudm4olhJKdFvYpZ4sIb4lssW4sazZg+VUoxcfAcAPu5wNy7dkIgw\nKKYKxaCYyATsQkFj6QtG0TkQTqlmticQhS9cOd0RfBHN1LrVXUd9w2qIB2uMdx31jbqvyh684+IC\nO6pkLJ8gMoE/olXk5W8aX5c/kna9bJc/CquiwGUv/4VgZrdHW3t6y6iftbbUjVlXDMR2a6txMS80\nUjDKE3yqXHxFIDKBqrOEgt7VORDOKOiTUuLYQBgRrbyzdWFVL/jfC4O/sYWjfB2jysWgmMgkLKEg\nIBbwZdO/2pASx/oj0Mr48n4xbKIRVg1u+zxCRNOhGeX7/44oGVOCYiHEb4UQnUKIt8wYj6gUcSMP\nAoBeExbMaYYRW5xXhkGbphsIFEHLLykl62dHYJaYKp1ZmeK7AbzPpLGISpKqG2V/2ZsmFlZ103q8\nRjUDnb6IKWMVE19YM2WzDjMEiyA4LyZBlSf2VNlMCYqllM8B6DFjLKJSFojwTbaSmZElThSMamVV\n+xrrEVw8vw83qXiXlBJhlZliqmysKSYyEeuKK5eZWeJE3f5o0WRWsxWIFlfNqmbw6s6gkKqXzf8z\nokzlLSgWQtwkhHhNCPHa8ePH8/WwRHnFEorK1RfMzeIxVTcwECqPk61iWGA3UpBXdwCwlIQIyGNQ\nLKX8lZRyuZRyeVNTU74elijvWEJRecKqntMyh75Q1NSNLgohoukIF+HCNm5pHMNSEiKWTxCZjiUU\nlSdXWeJBuiFNr1fOt2LNdkdUveRPOLLFHf6IYsxqyXY/gJcBzBdCHBJC/JMZ4xKVIlU3ijIjRrmR\n6yzxIF9YK/iGF5nSDVnULQvLaTFjJtiajijGlG2epZTXmTEOUbkIRDQ4beW/VS/lPks8SEqJnkAU\nU2qceXk8M/mLqA3bWEJRHV6nrdDTKBiWThDFsHyCKAe4aKUy5CtLPKhUW7QNhLM/cVi3qR1t7b3D\nftbW3ot1m9qzHi8YjXVeeGlvF+58dm/Wcy01DIqJYhgUE+UASygqQ38BuimUWou2YFQzpV51/hQv\nbt2wYyiQbTvQg7sfeQoXRR9H7Ws/gRLpz3g8Q0o8u+s4PnNfG5bMqMl6rqUkrOowSuj/E1EumVI+\nQUSjsYSivGm6UZBFlapuYCCsocZVGpf7zTpxaG2uxQ/PEXjtkR9hfs0+vKe/DR9AP7Aldrvr4As4\nuuZeSGtq5SWtLXW45bKFuHXDDly+dCoe2dqB2z90Cs6c02jKfEsFT96J3sWgmChHAhEdDVWFngXl\nSqCAl5z7glF4HVYoiijYHFIR0dLf0GTdpnbMn+JFa0vd0M/a2nsxafOPcP6R3+J8AIf6GnG4fgXk\n0vMQnroC9q63MPmJz6DpH59F54V3AkpqJ6OtLXW4fOlU3LOxHR89Y2bFBcQAF9kRJWL5BFGOaAZL\nKMpZIVvv6YY0pU431zLJEo8qk2jvxYZHHsTZR+5C+/RL8D7cjl+2PoyP9H0Cz3kvhVp/IgLzrkT3\nWd9A1d5H0fDCLUCK5QBt7b1Yv6UDN6xswcNvHMHzb1fWxlLc2ploOGaKiXLIzxKKsqQbsuAnPP0h\nFdVOW9Fmi2PlJek/RyPLGp5+Yw8ed96BgKUZ1xxei39fcwpaW+qwrLkWt27YgVsuW4jWljr0L7sJ\nlsAx1L5xJ3T3JPQt//yEj9PW3jvs+GXNtfjc/W34RQWVUIRVo6Tq04lyjZliohziFrLlqRh67uqG\nLMhCv1QNZNGG7d2yhgO4o/YeuCLH8UDLLUMB8eB9brlsIXYd9Q0d13PmV+GbdxXqX/k+vNvXTfgY\nu476hgLiwfG+fcXJ2HoovQV7pawUO5kQ5RIzxUQ5NFhCwWxxeSmWYKI/pKLaZYOlyLLFUkr4sijv\nGCxr+MG8HVjc/iTemv9ZvGf1xaPu19pSN6z2GELB8dU/hCXUjcZnvgTd3YDgrAvGfIy1p7eM+tlJ\n02pw6eKpGc+71LCemGg4ZoqJcqwYsopkHt2QRdPX1ZDFmS32RbSMt04eLGv47movrur4CbrqT8EN\nu84a1aN4XBY7jr3vfxFpOhmTHrsZ9q7tKT+2lLJiAkXdkCW7QyJRrjAoJsqxQi7IIvMFiiRLPGgg\npGYcgOZKfxa7/O066sM3LjkRq7Z9DRACwUvvwNfWnDysTCIZaffg6KW/BxQbatruSOvxi+WEJ9cq\nJfgnSgeDYqIcK6bMImWv2E5yDCnRF4wWehpDst2sY+3pLXjv8XvhPPoajr/nu9CqZ6C1pW7McoeJ\nGO5G+OZfjao9G6AEu1I+rlKCRb4mEY3GoJgoD1hCUR5iXSdSD/jM3pp4PANhDZoJu8aZYSCU3f91\nx9HNqHv1x/DNuwqBeVdkN5fFH4Uwoqjefl/Kx0Q1o+gy77lQ6O4pRMWIQTFRHgSjma/Ep+IRSPPf\ncayeu7du2IH5U7ymzktKib4iqC2OakbWixAbXvo2dM8kdJ17W9bzUevmIjjjHFRv+z1gpD6vcs8W\nq7phytbbROWGQTFRHuhG5SzgKWfptthL7Ll714vvDOuLazZfEWSLs1305zj6Gpwdr6Jv2acgHdWm\nzGlg8Y2w+jvgfufxlI8p99ICvhYRjY1BMVGe+MMsoShlRoYnNolbCV++dGpOAmIgli3uzWKBW7Z0\nQ2ZdJlTbdid0Ry18C9eaNCsgOOt8qFXTUfPm3SkfU+5BcbjMfz+iTDEoJsqTQFSHUQG1iuUq3dKJ\nQYlbCa/f0pF6a7EM+CNawdps9QWjWZUI2fr2wr3v/zBw8kcg7R7zJqZYMHDyR+A6/CJsPbtTOkQz\njLJuV8ZMMdHYGBQT5YmUsujaeVHqUtmyeOTCurb2Xtzy1204e24Dbjxr9lApRa4CYyklegL570Sh\n6gYGsrwSUvPGrwDFhoHFN5o0q3f5Fl0Hw+JA9Vu/S/mYcg0cI5peEQsJiTLBoJgoj9iFojSlWjox\ncmHd0zs7AQCrF0wCMPbWxGYLRrW8t43rCWSXJVaCXaja+QB8Cz4I3TPJxJnFGK4GBOaugXfnAxDR\n1J77cu3OUO6lIUTZYFBMlEehqF7wxVCUvqCqpxT0jVxY9/yebtz6/pOG1RFn0nM3XdkGqekIq3rW\nQXjNm3dB0SPoX/ZJk2Y12sDiG6GoAXh3/jml+5dr8FiuGXAiMzAoJsozZotLTzpBX74W1k1E1Q30\n5WnRXXcW5RrrNrVj674jqH7zbgRmXQi1bm5O+jgDQGTyMoQnLUP1W3cDKZwwGFKWXbZYyvT6bBNV\nGgbFRHnmYxeKkiKlRDCNrGE+F9ZNpC+k5rwXrS+sIpJF4Dh/ihfb/34nLJE+9LV+Kmd9nAcNLP4Y\n7L174Dz0Qkr3L7egOKwa7JdONAEGxUR5pupG2b3ZlrNQiqUTwLubc9xy2cJRC+uch19G0z8+B+9b\n98ASPJ7jWed+0Z2UEr2B7LLRrTOq8Tn3Y9iCE3Hnvsac9nEGgMDcNdCd9ahJccFduZUalNvvQ2Q2\na6EnQFSJ/BENTpul0NOgFKRTW7rrqG/YQrrBGmPXm/diyoEfQBcWeHc9CPncfyI8dQUCcy5FYM7F\n0D1TcjL3QERDMKrBbTf/pb4vqEIzsstEe/Y9iqrgIWyfdRvueeUgbljZktNyE2l1YmDR9ahtux0W\n32Ho3ukT3n8wsyqEyNmc8olBMdHEmCkmKoBAhNs+l4p0Aom1p8eCusEuFG/sP47z23+CS/Z/Fy/J\nk/Hohc/g4Np/oO/Uz8MS7kHj81/DzLtPxdS/XAlr/4GczL/bb/6iO003st69DlKipu1O+D0t+P7+\nOTktN0lslec76QYAQGDjXUlrl6Usn50odUNmVepCVAkYFBMVgG6kV6dKhaEbMqNNHFpb6nDrRTPQ\n9LcbUbPl17gHl6Lz0ruxeE4L1IaF6F3xbzh03VM4eP2z6FnxJdh7dmPKox+HiAZM/x1UMwLYEXqC\nURhZBtrOIxvh7HwDPwlchK+tOTmnfZwTW+Vp1TPQXbcMyu6/p1S7XC5dKILskU6UFINiogJhF4ri\nl2mW0Nq/H2c9cx3OwFZ8Wf1n7F72FSyb1TSqs4JaNxd9yz+PzovugK13N5qe+kJKnRHS1Rc0b9Fd\nRNNN2bK85o1fImitRfecq4Z+Nlhu8tTOTlM7UIxslff7nkVYgP1YXhdMemy5ZIrLJbgnyiUGxUQF\nEoxyZ6lil0kgYe/cgul/vgwerQcfjn4F3fPWYv2WDjzw2sFxOyuEms9FzxlfRdXev6H29Z+bMfVh\nDCnR0RfOOjBWdQPH+iNZz8fiOwz3gScRWXoDzl7UPCo7/MKebtM7UCS2yrMsuBgA4Nn/RNLjoppR\nFn+n5RLcE+USg2KiApFSMltc5NLuEiIlGp/7KiKw42rtv9B6zmXYtL8Xp8+qw53P7sP1pzePu5Cs\nf9kn4T/xCtRt/B5cB54yYfbDaYaBjr5wRuUgQCw47OgLZ724DgCqt98PSAnfog+NyuLmqgNFYqu8\n3+60weeZCff+f6R0bKkHlGGVJ+BEqWBQTFRADIqLl6obaWdWXYeeh/NYG56d/FHcuGY1rl7ejMuX\nTsUTOzpx/sJJEwcmQuD4qh8g2rAQkx7/F1j79mX5G4ymGQaO9qcfGEc0HR39IVMCYhgavDvuR6jl\nvdCqmwHkfsOTUa3y1izCQ4HFcBx6ASLqT3p8qZcecP0CUWoYFBMVUETVM87cUW5lkh2sffUn0Kqm\nYv7FN6O1pW5YdnLT/t6kJQHS5sKxS34LKBZMefSfUgrY0pVuYBxWdRztD5uWaXTvfxLWwFEMnPTh\noZ/lesOTwVZ5g8F2a0sdZp7xAVgMFa6Dzw3rTpE4p8G65lLvK85FdkSpYVBMVGDMFhencJrZNefh\nl+HqeAV9rZ8GLI4JN/KYiFbdjGMX3gFb3x5EH7gJbQe6h91uxjbIqQbGZgfEAFC97V5onikIzjof\nwMQbnphlsFVeouZlq6E7auHZ//iw7hSJcxo8iVF1o2RPXrUSnjtRvjEoJiowM1byk/nSzRTXvvZT\naK4m+BZdB2Ds7GTixh4TCTefg54zv4b5fc9i14afjhusZUMzDHT0h+CPb/ARiuoIqzoiWuzqRTCq\n4Wh/OOvWa4msAwfhan8avoVrASW2oUg2z1NWFCuCM1fDvf9JtM6oTlrXXKolFMESz3IT5RN3tCMq\nMM0wcrbrGGUmoiVfmLRuUzvmT/GitaUOjqOb4T70PLYu+iI2vH4ca09vwdrTW0Yd09pSl3K9bP/S\nm+De/yS+2PkgznvkHKxaNhfrt3SYughNNyQ6B8KmjJUK7/b7ACEwsOj6oZ9l+zxlIzjrfHh3/wWO\nY6+jteW0obrmsXbWC6oaamDL+ZzMVqrBPFEhMFNMVASYLS4u4Wjyy82Jl9zrXvsJIrZa3LR9qXmt\nxIRA91m3wK7247tNj+VsEVre6Cq8O9Yh2LIq6fbK+RJsWQWpWOHZ/0TSuuawasAosQ4OUkoGxURp\nYFBMVAQC7FlcVFIpnRi8zP/AIxvgPvAUfqlejH9fc4qpQWu06WQcaH4/Tj/+AD7bas3ZNsj54D7w\nD1iDnfAlLLArNOmoRmjaSii7/560rrkUt3wOq4ap5S9E5Y5BMVERYM/i4iGlTLnbQGtLHW6p+Rv6\npRsDiz9nLbRzAAAgAElEQVSWk1Zinzj4PiiKFZ9U783ZNsj5UL3tXmhVUxGcubrQUxkmOOsCVPv3\n4XurPEnrmkuttRm7ThClh0ExUZHwhdVCT4EARLTUs2t739qIRf3PY/OUa/HAWwM5aSV285qz4Tvl\nZlTtWY+V9n35WYRmMutAO1ztz2Jg4fVDC+yKRXDWBQCA0yKvDPt5a0vdqHrnUitFKLUgnqjQiuvV\niagQpISt921YfYdhCfdCCffCEumDEu6FEulHtPEk+BZcA8NVn9NpRDUDEU2Hw2rJ6ePQxFINfNra\ne1H17A+gWj044bJ/wy2d0vTd2AaDsr4pn4J32x/Q8OKtaL3q4ZKrK/Zu+wMgBHyL1hZ6KqNoNTMR\nrZ8P9/4n0L/sponva8Tam9mtxZ9PymTzGaJKx6CYKpYS7kPV7ofg3f4HOLp3jLpdd9RC2tzw7v4L\n6jd+D/45l8J38ocRnroCECInc/KFNTiqGBQXUqp1o13vvImrsBF9S/4FhrMWrS0YyuKaHbRKexV6\nV/w7mp75Ejx7/4bA3MtMHT+ndBXeHX9EcOb50KumFXo2YwrMugC1bXdACffBcNZOeN9QVC+JoJhZ\nYqL0MSimyiIlnEc2wrv9Pnj2PgpFDyPStARd596GSONJMJx10J31MBw1gBILTm3dO1G97V5U7XoQ\n3rcfQrTuRAyc9CH4FlwL6ag2dXqBiIYGjx0iR0E3TUxKiUiKGx1cp6+HtDqHZRdz2UrMt3Atarb+\nFvUvfweB2RcCFntOHsdsnncegzV0HMeLaIHdSMHZF6Du9Z/D1f4MAvOumPi+JdKardRKPYiKAYNi\nqgzSgOftv6Lu1R/B3rcPht0L38Jr4Vt0PaJNJ094qNqwAN3n/hd6zvhPePY8gupt96DxhW+i5s27\nceSKB0zNfumGRCCqo8rBP81CCKk6ZCr1xLoKz75HEZhzKQxXQ+4nBgCKBd1nfg1TN3wYNW/enfRS\nf7HwbrsXatV0hFrea+q4FkXAZbPAabdAEQLHfZHU/u3GEJnUCs3VCM/+x5MGxYOt2RSleE9cS7FT\nBlEx4DsvlT3n4ZdR/9K34ezcgkjDInSe92ME5qyBtLnSGkfa3PAvvBb+hdfCeehFTHn045j20NU4\ncqW5gbEvrDIoLpBUs2uuIy/DEulHYM6lOZ7RcKGZqxBsfg9qX/spfAuuhuEs7tpiW89uuA89j54V\nXxq68pINl90Ct90Kl80yqoRBADiW6UYkigWhmefBve/vgK4ClvEzwYMBp6eI/0ZTPrkjomGKvzCK\nKEO23j2Y/LcbMe3hD8Ia7ETneT/B4Wsfg3/BNWkHxCOFZ5yFjsvvgyXcjWkPXQ2L73BG46zb1D6q\nY8FLe7pw+9N7spofZSbV7Jpn799gWN0INZ+T4xmN1n3W16FEB1D72v/k/bHTVbP1NzAsTgyYUDrh\nsFkwtcaFGpdtzJpej8OKJq8j4/EDsy+EJToAZ8empPct9nrdYp8fUbFiUExlRwl2ofGZL2PG/avh\nOvIyuld+BQc/9Dz8C64GhHn/5SNTTkXHmnhg/HBmgXHirmhArKPBrRt2YO6kKtPmSanRDYloKvXE\nhg73vscQnLka0prdyVUm1IaF8M//IKrf+h0s/iN5f/xUKeFeVO36M/zzrsy6xEQIgaaq5AGv12lD\ngyezwDjUfC4MiwOe/Y8nv2+RB53FPj+iYlW813+I0qWrqHnzLtS++mMoWhADJ38Evad9Iac1n5Ep\np6BjzX2Y+sj1mPbw1bEa4zS2sB3cJODWDTtw+dKpWL+lA7dcthALp5q7gI+SS3XDDsfRzbCGjiMw\n55Icz2h8vaf9K6p2P4S6V3+CrlXfL9g8JuLdfh8ULYz+pf+U9Vj1HnvKHR9q3DboUqIvGE3rMaTN\njfD0M+BqfzbpfTWjeNsnhlU9763YRNQX6+Sz80+wBo4CUgKGDgEDkLEP3T0Z4WkrEJq2EuFpK6BX\nTc3rHIlSwaCYyoKr/Rk0vPAN2Hv3INiyCt1nfxNq3dy8PHZkyinouPx+TFl/PaY9/EEcueLPaQfG\nly+dins2tuOGlS1obamDqhsIRXW47MX3pluuUg2KPfsehVTsCM48L8czGp9W3YyBkz6M6rd+j77W\nT0GrnV2wuYwpfoIanHE21IaFWQ3lsltQ40qv20O9xw7dkGlviBOacQ4aXvo2LP4jSdcJhKLFGRQH\n8rUzppRwHNuM6m33wbNnPRQthEjDQgSb3xO/IicgFcvQ17b+A6ja/RCqt90DAFCrZyE8bQWCzeci\nOPtCSJs7P/MmmgCDYipp1r530PDit+DZ/wTUmlk4eundCM48P2d9hMcTmdyKo5ffjynrr8PUR67H\noWsfByypXcZta+/F+i0duGFlC9Zv6cCy5lq0ttTBF1EZFOdRSvXEUsKz7+8ItpwLaS9siUvfqZ+D\nd8f9qNv0Qxy/8OcFnctInn1/h9Xfga5zv5PVOIoQaEyhbGIsTV4HDCnTChJDzecCAFwHn4d/4bUT\n3jcY1VFbhHFczreLlwa82+5FzZu/g71nJwybB/55V8G36HpEJi2d+LXX0GDv2g7XkY1wHnkF7nce\ng3fnH2HYPAjMuQy+BR9EeNpKU8vciNLBoJhKkhLqRu3mn6PmzbshLTZ0n/HV2GXaFAPRXIhMXobO\nC2/H1A0fRu3rt6PvtC8kPWawhnhwF7RlzbVD3586sx6Gp7hbP5ULI8V6YvvxN2HzHULvaf+ah1lN\nTPdMQv+Sf0bt679A/ymfRrRxUaGnNKRm66+hVs9CcNb5WY1TX2WHzZJ5gNTgsSMYTb0TQ7RhITRX\nE9wHn0saFIdVHbohYSmiv89gVINu5LDrhB5F01NfhHf3XxCetAzHV/03/HMvT/0EUbEiOmkJopOW\nxFoKSgPOI6+gatefUbVnA7w7/wi1ajr88z8A/4IPQq2dk7vfhWgMPB2jkiIiA6h75QdouecM1Gz9\nNfzzrsTBDz2P/lM+XdCAeFBo5ir4565B7eafwdr3TtL77zrqG7Yt8GCN8a6jvliWK5qnS6EVLqyl\nUTohLAjOviDHM0pNf+vNMOxe1L3y34WeyhDHsTY4j25G/5KPZ5Xxc9utqHZmt0mG1aLAk87VFiEQ\naj4HrkPPx2phkyi2XsD+cO5eL0TUhykbboB391/Qs/LLOPLBDfAtuj67KyZCQXj6Geha/UMcuPEN\nHLvgF1Dr56H29Z+j+Q/nYupDH0DVrgchtJB5vwjRBBgUU0kQagg1r9+OlnvOQN1rP0awZRUOXfc0\njp/3I+ieyYWe3jDdZ38TUGxofO6rsQUnE1h7esuoHdBaW+qw9vQWAHm4FEoAYhsypMKz7+8ITT8D\nhrM+xzNKjeGsQ3/rzfDsfxyOo5sLPR0AQPWW3wxtjpMpiyLQWGXOjn3VadYjh5rPhSXUDXvX9qT3\nDRbRSasR3/gnFyyBY5j2l6vgOrIRnef9GH2nftb0EjVpcyEw7wocXXMv2j/6KrpXfgVW/1FM+sfn\n0HL3cjQ8fwts3TtNfUyikRgUU1FTIv2oeeNXaL73TDS8fBsik0/BoWv+D53v+2XeFtKlS/dMQc+K\nL8F98Fl49jyS1VihqA4tzyvJK1Eqi+xsPW/HFnKecHEeZpS6/iX/DN3VgPqN3yv0VGDxd6Bq7yPw\nLbw2qwxiQ5UD1izKJhI5bRY4bKlniwd7T7sOPpf8vkXU+iwQ1XKyYYetdw+mPXg5bP37cfSSu+Ff\ncI3pjzGS7pmC/lM/g4Mffh5H3v9HhFreg+q37kHzuvMw7c9r4H3rd1DCPTmfB1UeBsVUlOxd29H4\n9JfQcvepaHjxW1BrT8CRK/+Co2vuQbRpcaGnl9TA4o8i0ngyGl74JkTUl9VYzBbnlpQSkRTqiT37\nHgUABE54X66nlBZp96D31M/BdfhFOA8+P/TzsTaGaWvvxbpN7TmbS/VbvwMMHf2LP57xGDaLYvqO\njtXO1MfTPVMQrV+QUlCsGzLlriW5lovXCUfHq5j24PshtDCOXPkgQjNXmf4YExIKwjPORueFt+PA\nxzaj+6xvQFEDaHr2PzHzrlMw+W83wrPnEZZXkGkYFFPx0FV43l6PqX+5CjP+eAGqdj0I/4lX4NA1\nj6HjygcRnrai0DNMnWJF13u/C0uwE/VZ1nv6clgnSLHSiVQybJ69jyI85VTonil5mFV6fCd9GFrV\nVNS/8r2hkp3xNoaZP8WbkzkILYTqbfciOPsiaDUzMx4n3XKHVFQ5rLAqqb/dBZvPgbNjU0rBVjH8\nfWrxFo5msnduxdS/roXurMORD6xHdNISU8dPl+GqR/+ym3Bo7ZM4dO3j6F/yT3Ac34rJj92MmXe1\nounJf4X7ncch1GBB50mljd0nqKCEGoSr/Rl49j8O9/5/wBLuhVrdgu4zvw7fwmthOOuSD1KkIpNb\nMXDyDah+8y74FlydcYZb1Yt3o4BykEqmzzrQDkfXW+g+8+t5mFH6pNWJ3tP+FU1P/zvc+59AcPaF\n424MM7KG3SxVux+CJdyb1WYdihDwmpwlBmI74nmdVvSmuKFHqPlc1G75XziPbEKo5T0T3jcQ0dBY\nZYfIcxvI4XMwNyAWagiTnvgMDGctjlz1MAx3o6njZ0UIRBtPQk/jSeg54z/hPPwSvLsfgmfv3+Dd\n+UdIxY7Q9DMQnHkegrPOg1Yzq9AzphLCoJjyzhI4Cvf+J+F+5zG4Dr0ARY9Ad9QiOHM1/Ce+H6GW\nVYBSHgFg78ovw7P3UTQ+82Uc+cD6jH8vf1iDo6o8npNik0rnCc++vwMAAkVWT5zIt+Aa1Lx+O+pf\n+jaCLe8BLI4xN4bJCWmgZstvEGlYhPC0MzIexuu05qwFYbXLhr6QmtJVgfC0lZCKHa6DzyUNimNd\nYnTTSz7S4Yukt0lJMvUv/xfsfXtx5PJ1xRUQj6RYEG4+B+Hmc3D8vd+Fs2NT7L3lwJNofOEW4IVb\nEK09AeFpZyAy+RSEJ7dCrT+RfZBpXKb8FQsh3gfgpwAsAH4tpfyuGeNS6RORATiOb4Xj2Btwdm6B\no/MNWP1HAACqtxm+kz6MwOyLEJ56OmAx/7JpoRmOGvSc9Q1M+sdn4d1+L3wnfzSjcfwRDfWewmaj\nypGUEpGEzhPrNrVj/hTvsOCxrb0Xp295GFWNJ2VVFpBzihXd53x7WJ/s8TaGSdd4z8uuoz6sPb0F\nVbsehL1nJzrP/5+suhLkonRikEUR8DgsKbUtkzY3wlOXw3XwWQDJrw74wmrBguKoZqTUYztVrgNP\no+bNu9G39BMIxxcdlgSLHeEZZyM842z0nP0NWPv3w33gabjbn4Jn7wZUb/8DAMCwexGetBSRyacg\n2rgIau0JUGtOgLS5CvwLUDHI+q9YCGEB8AsAFwA4BOBVIcR6KWXyfjZU+qSEEh2AdeAgrL6DsPUf\niH9uh7X/Hdj73+3Vq1bPQnjqaYhMWorQjHMQbViY953nCsE/70p4d6xD/cvfhf/EKyEd1WmPoRsS\nIVWH286LO2aKaAaMhMzhYB3uYJlBW3svbn/kJVyFN9G76IsFnGlqEvtkv+pZhVufCY65McxYgfFE\nge9Yz8vg9yLqQ/1LtyE8uRX+eVdmPHePw5rVRh2pqHHZUu7lG2o+F/UbvwtL8Dh0d9PE9413iTGr\nY0Y6zFxgp4R60PTUvyJavwC9K79s2riFoNXMwsCSGzGw5EZAGrD17YPj2OvxPtqvo/b1X0DId68S\naVXTEK09AWrtHGjeZuieSdDck6B7JkN3T4LhqKmI96tKZ8Y77OkA9kgp9wGAEGIdgPcDKK6gOBqE\n4j8Oi8mXmfJu6A1cjvgeEFLGfy7jP3/3s5BGrBm91ONfSwhDAwwVwlBjX+vxr/UIhBaGogYhtFD8\ncxBK1A8l0gtLuBeWUC+UcOxrYQyv09Pt1dCqW6A2LIR/wQcRmbQMkUlLiqa3a94Jge4zv4YZD1yM\n6u33ob/15oyG8Yc1BsUmi4zoTzxWHe7di9sh3pQInHBJgWaZnu6zvwV3+zOYt/kbuOXSu8bcGGas\noHiiwHei+uS6F78Na+g4jl16d1aXpWtymCUe5LBa4LRZUqojD8aDYtfB5/Hr/uUTZsqB2IK7Oo85\nvZXTYdqGHVKi6Zn/gCXch6Nr7oW0Os0ZtxgIBWrdXKh1c4daygk1BFv/Pth698Y+9+2DrXdfrDY+\nOjBqCMPigOFqgGGvhuHwwrB7Y1/bY19LqxOG1QlpcUJanZBWR+xrxQapWAHFCqlYIRVbrIxOWCAV\nCwAR/6y8W14nBCQEABELxIUS+zp2I+TI4HzcYH2cnxcquLdXATXFt1A5kRnvsNMBHEz4/hCA4msT\nsGM9Gh76JBoKPY8SZVhdMGweGM466M56qDWzoE9uheGqh+6sh+adAbVmJjRvMwxnbaGnW3Sik5Yg\nNP1M1Gz539hOX5b03zwDUR2GwW2fzTTWjmQj63Dnd/8slkGqn1eAGaZP90xGz8ovY/ZzX4Un/Az8\nuGrottaWunHLJ5ItzBurPtnWuwc1W3+NgYVrEZm8LOM5260KnGn0Es5GtcuWUlAcbTwZurMOrkPP\nYf6Jq8Y9YRjkj+Q/KA5FdWiGOaUTVTv/BM++R9F9xtcQbTzJlDGLmbS5EG08afTvKiWE6oc1cAyW\nYCcsgU5Yg8dgCR6HJdQFJeKDEh2AJXgctt59UKIDUKL+UckhGk3WzQI+v6XQ05hQ3tJOQoibANwE\nAC0tLfl62HdNXw7fBT8omp6S2YkHRUIM/z7xDFIknGVCgRRK7GxTKAlnpQqkYo+dvVrs8TNZG6TF\nDml1w7C5Ia2uWMaAl42y1rfsZkz920dQtWc9/PM/mPbxUkr4o1rWW9/SuyJjLLJLrMN97I39+Kay\nEb7FHyupv4GBk25A1c4/o+HFbyI4c1XKXVwmWpg3qj55Rg0u3voNSKsLPSu/ktV885ElHuSxW2BV\nlOTBpGJBaMbZcLU/h9bVtUk7eai6gbCq5y24B8wrnbD2H0Dj819HaPoZ6F92kyljliwhIO1eqHZv\nehtESQNCi0DoIQgtHP86DGFo8auy2vCrsohdsR26emvo8a3F5eirvglbjouRV4sxzsLRcReUmr/B\nS6rq6+pR7MvFzQiKDwNoTvh+Rvxnw0gpfwXgVwCwfPny/P+rNM5F2NEMX7jEyyeoZIVmrka0fj5q\n2u6Ef94HMgqy/GEGxWaJaDp0Y/hL0ciSgQucO2HZGMU2xzJMzfLxvE4b6j12+CMaBkIq1FzuVKhY\n0PXe72L6A5eg/uX/D12rvp/SYeMtzBv5vCxrrsWzj/weH8Az6Dr7m1l1KLAoIq+L1IQQqHZZ0RNI\nntkLNZ+Lqj2PwNazG60t85N28hgIq3kLiqWUCJgRFBs6Jj35eUhhQed5Py2bzj95JxRIm4sL9iZQ\nV+8u9BSSMmNVwKsAThRCzBZC2AGsBbDehHGJyosQ6Fv2STi6d8RXtacvrOq5DaYqSFgd/TzuOuob\nlgFcHH0DhrDiZS3z0gkhBBq9DjR5HbAoAjUuG5rr3Zhc7YTLnrsAJNp0MvqX/BOqt/8Bjo5Xk94/\nMfC98azZQ5nRwdrZxOfllGkufMd9H447Z2Pg5I9lNc9qpy3vXVVSDcJDM84FALgOPTfqhCFxt8DB\n3QMDkViJEwC8tLcLdz671/zJxw2EtGGLRDNVve0eODteRfe5t0H3TjdhZkSlK+ugWEqpAfgMgMcA\n7ADwJynltmzHJSoXidvt+uddAc09GcrLP8t4u13TFtZUuMgYpVRrTx+eAXQdfhHRyctw1RkLR903\nFVZFwdQa55jZfY/Diqk1Lkyvc+VsAWXv6f8GrWoamp75MqBPfJVsZOCbuDBv5PNS88YvURU8BO3C\n72TVSnFwU418s1oUOFLI6GrVMxCtPQHa7ifHPWEA3l2k+PqBHvijGl7a24XP3NeGJTNqcjJ/w5Do\nC2VfwyrUIGpf/QlC087IqnMIUbkwpX+MlPJRKeU8KeUcKeVtZoxJVC6GbbdrcWDHzOsxuWsjTncd\nymg8M1swVbKxMsWJRGQAjs4tCM04K6PxXXYLpte5kl5Od1gtmFztgN2a+ctx4onXoLb2Xtz/Rje6\nzr0N9p6dqHnjlxOOMTLwBWKB8WB3hUEW32HUbv4ZAidcglDzuRnPGYjX9xagjdngYw8a7/lbt6k9\n1pqt61V88+I5Y54wJH5/64Yd+PHju/GZ+9rw8+tbceac3Gx80R9SR5X+ZKJm629gDR1Hz8r/KKma\neaJc4bYuRDmW+IZ514vv4FM7lkK1uLHi6P0ZjTe4oIcyp+pG0oVWriMbIaSB0Iyz0x6/xmXD1BoX\nLCl2ChFCYJLXCSXDwGTYiRfeLYWYP8WL4OwLETjhEtS/8j1U7Xowo/ETNbx8GyAlus+6JeuxcrlZ\nRzKJ2fmJnr/QjHNhN8JYYR9eCjHyhGFwkeJdL+3Hdac15ywg1g2J/lD2a2OUcB9q2u5AYNb5iEw9\nzYSZEZU+BsVEeZC4qn/VsrkInPwhVL39V1h8o9akpsSUBTYVLJWTCtehF2BYnIhMPiWtsW0WBQ1V\njrTnZLcqaKjKrKXXyBOvkZt0dJ73E4SnrUDTPz4P7/b7MnoMSInazT9D1dt/RX/rp6BVNyc/ZgL5\nbMM23uMPbhYy0fMXmnEmpLDA1T7xOoDEmuM/bGrHS3u7cjLvvmDUlFrimrY7YIn0o3fFf5gwK6Ly\nwKCYKA9GLtJ5qTHWQL5my/9mNB5LKLIzVn/ikZyHX0J46mlpb2JQ4848++l12lCVYY1t4onX5Uun\nDiuFkHYPjl52D0It70XT0/+O6q2/TW9wPYqmp76I+o3fhf/EK9C7/LMZzTGRtwi6qLgTSijGe/6k\n3YvIlFPgPvjcuOOMXKT4zTUn4TP3tZkeGGu6gQET1hRYAp2o2fob+E+8AtHGRSbMjKg8MCgmyrGx\nVvV/5ak+HJz2PlRvvw9KpD/tMXVDIhRlCUWmRu5kN5IS7IKjewfCadYTWxUF3izbizV6HBltdzxR\ndwQAkFYXjl7yGwRmX4TG57+OmtdvT2lcJdyLqY9cD+/OP6J3+RfQecHPAUv6mfBEQuS3Ddt4PAlz\nmOj5C7asguP4VliCx8ccZ+QixSUzavCDq5dg66H0/7Yn0htUIU3IEtdu/hmEHkXP6f9mwqyIygeD\nYqIcG29V/+O110BRA/BuuzejcX2lvmV5gWi6kbStnevwiwCQdj1xjTv79mKKIjCp2pHWOBO1UxvG\n4sCxi34J/9zL0fDybajb9MMJmvwD1r53MO3By+Hs2IzO8/8HvSv+zZQFWR67JeV661xy2mLzSPb8\nBWeuBgC42p8ec5yxFimeOMmLm845wbS5RjXDlD771oGDqN52D3wL10KrnW3CzIjKR+FP1YnK3MjV\n+0AsMEbL+Qj2n4OaLb9B/9J/Tjv7FozokFUy7z1eS11YS97n2XXoRRh2LyJNi1Me16IIVJvUXsxh\ntaDeY0e3P5LS/SdqpzZqkwmLDZ0X/BzS6kTdqz+CJXAU4WkrobuboLmboLsnwXDWwdnxKiY/+nFA\nCHS8fx3C01aY8rsBxVE6MchttyZ9/qKNJ0FzT4L7wNPwL7gmpXFV3UB3IIomb3ZZ9UF9QXO2Ea57\n9UeQQkHvaV8wZTyicsKgmKiA+pfdhKkbboDnnccRmLsmrWMNKRGI6kVxGbqUpFJ24jr0AkLTzgCU\n1J/bGpe5m1DUuGwIq3pKiyrHO/Eaa9c1AIBiwfHVP4Rh86DmzbtQPWLxnRQWABJqzSwcvfT3pmYU\nbRYlp5uWpMvjsCR//oSCUMsquN/5P8DQUv5/4Qur8DgsWfehjmi6KesIbD27UbXrz+hf+gnoVdnu\n0UhUfvhuSlRAoeb3QKuaBu+OP6YdFAOxLhQMitOTrPOEdeAgbAMH0L/k4ymPGcsSm5/9bKxyIBTV\nTek2MIpQ0H3uf6FnxZdgDXbCEjwe+wh1wRLsBKREf+vNMJzjBNYZKrb/ry6bBYoQSZ/j4MzV8O78\nIxxHX0dk2ukpj9/li2J6XXblIr0Bc0ql6jb9N6TVjb5TPmPKeETlprhenYgqjWKBb8E1qH3tp7D4\nDqe9zWowqkM3ZFHUZ5YC3ZDJ64kPpV9PXO20QcnBv8HgttC9Jl06H4t0VEN1VEOtm5uzx0hUiB3s\nJiKEgMtuSZqRv7fzBHxBWOA+8ORQUDy4BfZYmeZBmmGg2x/BpOr0upgM8kc0BKPZZ4ntnVtQtfdR\n9Jz2RRiu+qzHIypHXGhHVGC+BddAQMK764G0j5VSImDCG2alSK0V24vQXI1Q6+enNKYiRE43oahx\n2WBVyuOl2lXAHewm4k6hnGPWjKnYLOdD7HkCwPANPpLxR7SMyh8Gwio6B8JpHzeWutd+Ct1Ri/5l\nnzBlPKJyVHyvTkQVRquZidD0s+Dd8SdAJl8ENpLfhL6llSLpph1SwnXohVgrthTrg71Oa04z9Yoi\nUOspnoVp2SimBXaJ3HZr0nrw1pY6OBe+D7UDu/DnpzeN2iAlmW5/JK2tmfuCUXT5UltomYytdw/c\n7zyOgcUfg7QnD+KJKhWDYqIi4Fu4FraBA3Ae2Zj2sWFVh5akJIBiki2ys/XugTXYmXLphBCx8oZc\n8zqsGfUuLiYWRcBTRAvsElkUAact+fNbu/QSAEDXlkdHbZCSjG5IHE8xyO32R9ATMK9kpuaNX0Fa\nHOhffKNpYxKVo9J+lSUqE4E5F0O3V8O7/f6MjucOd8mlVk/8AgAgND21TTu8TmteygGEEKj3ZLYF\ndLHwOJJnYwsplQ4Rm/yTcRQN+Gjj7jE3SEkmGNXQF4yOexIrpUSnL4z+kHk9yC2BTnh3PgD/gqth\nuBtNG5eoHDEoJioC0upC4MT3w7P3UYjIQNrHMyhOLpV6YtfhF6F6m6HVzExpzHxkiQd5HFY4bcWZ\naU1FsS2wGylZFrutvRe3/m0ngi2rsTC4Gd+4ZO7YG6Qk0ROIor0niIM9QXQOxALgsKrDMCSODURM\nLyOawcQAACAASURBVIeqfvO3gKGib9lNpo5LVI4YFBMViYFF10HRw6h6+69pHxvVDEQ0bvs8kaT1\nxIYO5+GXEEpxa2eHzZL3koZSzRbbrQoc1uIO6K0WBXZr7N9z3ab2UcHu0zs7cfbcBrhPuhiK6sdK\n69tDG3xkQtUN+CMauv0RHOkLYX93wJQuE4lENIDqt+5BYM7F0GrN212PqFwxKCYqEtGmJYg0LIR3\nx7qMjg9EGBRPJFk9sb1rGyyR/pTriQtRH+u0WeApsj6/qSjWBXYjeeIlFPOneIdlgdvae/H8nm6s\nXjAJoRlnQyo2uA88hdaWugnbsRWad8f9sET60L/sU4WeClFJYFBMVCyEgG/htXB2vgFb9460D2cX\nivGlU08cnn5mSmNmu0tZpurc9qKuzR1JCFF0G3aMx+2InegMbvN864YduOvFd4Z1mpB2D0LTVsJ9\n4KmCzHGsLHZbey/WbWoffkddRc0bv0Jo6gpEppySxxkSlS4GxURFxD/vA5CKDd4df0z7WM0wUtrC\nuBIlLZ1ALCiO1s2D7pmc9L62hEvt+Wa3KkVfn5vI48huN7d8clgtQz2hW1vqcPnSqbhnY/uoThOh\nmath790N68ChvM9xrCz2WP2SPXs3wOY/jP7Wm/M+R6JSxaCYqIgYrnoEZl8I764HAT39lky+iHmr\n1stJ0kV2ehTOjk0IzUgtS1zoEoY6tx1KiWSL87kY0QxOe+xtsa29F+u3dOCGlS2jOk0EZ64GALja\n858tniiLPURK1LbdgWjdiQjOOj/vcyQqVQyKiYqMb+FaWMI98Ox/Iu1jA5HYKnYaLlmm2HH8TSha\nCOFpZ6Q0Xio7oOXS4PbPxc5psxT9AruR3HbrUPb1lssW4sazZg8FoYOBsVo7B2p1S8FKKCbKYgOA\n69DzcHRtQ/+ymwHBt3miVPGvhajIhJrfA80zJaMFd1JK+Lnt8zC6IRHVJq4ndh55BQAQnrYi6XhW\nRSmK1mg1LlvRlyXkcvvrXHHZLNh11Dcs+zqYnR3qNCEEgjNXw3XoBQjNnG2Y0zFRFhsAatrugOae\nDN/8K/M+N6JSxqCYqNgoFvgWXANX+zOw+DvSPpwL7oZLpZ7YeWQjorVzoLubkt53cDFWoSmKQK2r\neFu0WRWlaHewm4hFEfjoWbNHZV9HdpoItqyGooWGTqjyZaws9lcfegsPvHYQAGA//hbcB5/Djpbr\nsG7zsbzOjajUMSgmKkK+BddASANVux9K+9iwqifNjFaSpPXEhg5nx6sIT1uZ0nieAnWdGEu1q3i3\nf652FfcOdhNxpXAlIDz9TBgWJ9wHnszDjN41Vhb7xrNm4a4X96OtvRc1b/wSqsWNT+1cNmrxHRFN\nrDhfTYkqnFY7G+Epp6Jq918yOp473L0rWabY3r0DluhASqUTihBw2ornZVMIgVp38ZUoCCFKpjfx\nWFIJiqXNhfD0M2NBscxfHf/a01tGZbGvXt6M2648Gb9+5Fm4d/8V9+nn4Qtrlo+6HxFNrHhe3Ylo\nGP+8q+Do3gF71/b0j2UJBYD06olDKWSK3Q5L0WU/vU5bwdrDjaeU2rCNxWlTUuruETjhItj698N+\nfGvS+6bcXzhDrS11+EbjU9AlcGzRxxkQE2WguF5JiWiIf+4aSMWaUQmFZhimbxlbilKqJ+7YCNU7\nA7p3etL7FlPpRKJst382O2Arhc4YExFCpLSYMjB3DQyLA96dDyS9b6r9hTO17e29WNr1CN5suAh/\n2KGN+vckouQYFBMVgbGCks3HFezxroiVUMj0a4SZLU4hKJYSriOvpFQ6IYRI6bJ6Ibjt1qw6YpgZ\nsJViG7axuFJYJGg4ahCcfRGq3n44aV/xlPoLZ6itvReHH/8fOKFi0kVfGtVCjohSw6CYqAiMF5T0\nzHk/rIGjcB5+Oe0xA1H2LE62yM7WtxeWUHdKi+xcNguUIi4JyCZbbGbAVopt2MaS6gmQb8HVsIR7\nU1pwl6y/cKb2HT6Gj9seR3D2RVDrTxzdQo6IUsKgmKgIjBeUTFp+FQybJ6MSikrvWZxaPfFGAKn1\nJy6WVmzjcdosWe20Z0bAZlUUVBV4tz+z2K1K0s4e6za142W5GJp70lAJxURlJ8n6C2fqRvfzsKsD\n6Dvl00M/G9lCjoiSY1BMVCTGCkqkzYXAnEtQtXdDRpsE+Cq4hCK1/sSvQHM1Qa05Iel9i7WeOFGd\n257xQkAzArZqV/E/R+lIVpIyf4oX33r0beybegncB57Etrf3jlt2kmyXvIzpKmrf+BVC01YiMuXU\n7MYiqnAMiomKxHhBiX/eVVCiPrj3/yPtMSMV3LM4aX9iKeE68jLedi5G28G+YTeNzPY5baXRTcFu\nVTJa5GZGwFbqbdjGkmw778ErPF/ZuxjC0LD98bvGLTtJuktehqrefhhW/xH0nfIvWY1DRAyKiYrC\nREFJaPpZ0NyTMyqhACq3Z3EoOnFQbPUdgtXfgcj0lUkXmZVClnhQnduW9oYeZgRs1U5rSZw4pGO8\nuuLEhbGtLXVYtGwF3jRm4Wrrc+OWnYzVXzjrEgdpoPb12xFpWIhQy6rMxyEiAAyKiYrChEGJYoH/\nxPfDfeBJKOH0L7X6wxpkHjcX+P/bu/PwuOrzXuDf9+yza7e8IBuIMWaJERjiQhLCkjQJxiwNhZDQ\nJu3TXm7aJmn7NG2SXt9bsrZp0vQJ6ZKbW56bQHGaAAnQcBMIhDQYYwwGYmPEZiMvMpZtSR4to9l+\n94+RFEmWZjlzZubMzPfzPH6wxZwzP3ukOe/8zrv4QTKdRSpTXD5xx9mXFSwy83s+8Wwigs6IXdIx\n5QZshqahNejfkdNuadrCrdlmF8bu7B/Cvc8exI/UpTgt/Rpe21W9sc/BfY/AGnoZI70fA3zWP5uo\nHjEoJvKBQkHJ6JrfgmRTCL36YMnnTmezTbdbXGiXGMjlE2fsGJLtZ+YtMjP1wgVXfuOYelV7BbeF\nLV935ijHQrvF0x9aN/9oNz5z3y4AwLr3/T6yYuDQ43dUrRVay85/QiqyAqOrN1Xl+YgaXX290xP5\nXMg2sKwlgJag5emUsWTH2Ui2rnadQjEykfJsLfVgPFX4Q4BzaBsSSy8CRMtbZFZMv1o/agtZVQnm\ng5bRMB0nFrLY69/b04o13RFMprO4/vzlOHv16ZhYdQVutLfi5YHKB8X2oe1wBp7GyHm3Alrj/vsT\nVRODYiKPhG0DXREbjqmjLWRhRWsQPW1BdERsBC2jvPHAIhg943oEBp6CcWJ/yYcn09midk8bQTar\nkEjlT53Qx47AGtmLxLINBYvM/DqwoxARQUe4tDQKN8/RHm68tInZFiuy3Nk/hNcGx+Z8kIqfeQOc\nyWP4yJK9FV9X6zPfQMZpQ3ztTRV/LqJmwaCYyANh20BX1Dkp8DV0DVHHRHfMKTt4GD3jutxzudwt\nHp7IP3GrUUykMgVzqJ2BXN5nYtnbChaZlTMprtYCll7RjhAtgdKL+urR/A9Gi32Q2qpdgIzTikhf\n4bHPZa3njccQ7H8Uw70fgzIDFX0uombS+O9mRBUWdnIBcSFRx4RdRoCVjp6CiaUXTY19Lr1wbiKZ\nwWS68XeLx2ftiC80Pntn/xD273wEWSOIyY5z8uZzW4ZW9x0V2kMWDM37t3pT19ASbKwWbItx5qVQ\nLPZB6qUjCYyuvhbBvT+Flhhe6FTlyyTR/sv/iWTLaRhZ9/uVeQ6iJsWgmKgMEcdEV6RwQDytvYxR\nvECuZ7E19Aqso7tdHT8y3vi5xbPTRBYbn/2WxAtILF0P6PmDunpNnZhN0wQdEe9THDrCdnkpQXUk\nOO/7IN8HqfiZN0DLTCL06gMVWUvshX+DNfwajr39bwC9sVNXiKqNQTGRS9GAWXLrK8cs73b22Fs2\nQmkmwn0/cHd8MlOwVVk9S6QySGd//fdbaHz2596zDLETrxQ12rlei+zmC1qGpy3TwrbRMP82xTBK\n6ECS7Hwrkq1nVCSFQh87gtan/wFjK6/AxMrLPT8/UbNjUEzkgmPqrouY2kKW61vyWacV4yuvQPjl\nHwLZ0tusKaVwooE7USxUTDi/3dpF+ssQKCSWbch7LhGBYzRO4NcastDuQeGdJoK2Mu941KNC0+1m\niCB+5m/DOfzMTC9sr7Rt+zIkM4ljb/9fnp6XiHIYFBO5UE5QoGuC1jKOj6/5LRgTgwjs/y93xyfS\nyGQbc5jH+AKjnee3WzvR9ziUZmGy67y857IMreF678Zc3N2YTRNBV9SG0QTFdfMFS5hqeOLc30Uq\n2oOOx/4Ckk548vz2mzsReel7GDnvD5FuOc2TcxLRXM33zkZUprBtlN2RoJyiu/FVVyBjtyDcd4+r\n47NKIZ5ovN3iTFZhcl5QvFCXgMnXfomjLedCGflzwRshn3ghEcfEkgU6pRRimzqWtwZKCg4biWNq\n0Ir8N1NmEEff9bewhl9Hy9P/UP6Tqyzaf/E/kA4uwdAFHy//fES0IAbFRCUQKW+XdzbXRXe6jbG3\nbERo70OQ5KirU4xMpBpu9PN48uR0kvldAs7vtnC27MXL9rkFz9eoQTGQGzLTHXWKDvJiARPLYk5T\ntF9bjIiUlEc9cco7ET/zRrTs/GdYg7vKeu7wS9+Hc2Qnjl/8GSgrXNa5iGhxzfsOR+RCxDE8CwzK\nKbqLr/kAtHQCodd/7Or4TFYh3mCjnxfKJ57fJcAZ2A5NZXDa+vfkPZeIwDEb++0xYOnojjl527Xp\nmkz12G6eThP5lFpceOySzcg4beh89M9d1QAAgEyeQNuTX0Ki+wKMnnG9q3MQUXEa+12fyEOaiKcV\n/ID7orvJ7vVIRVci3Hev6+dupPZsSqk5/YkXEzi4FUozkVh6Yd7HOabWFEGgY+roaQ9iZXsIy1oC\n6IjYaAlaCNkGgpaB5S3Nmy6xkPmt2QrJOi04dukXYB/dhdhz/+rqOVt3fB36xFEcfcfnAOElm6iS\n+BNGVKSWoOn5IAddE8QCLnaLp8c+H/gl9NEBV8+dymQbJrc4kcoiW0Q6iHNwKyaXnAdlBvM/roG6\nThRD1wSOqSPqmGgLWVgSdXK7yE2cLrEQQ9dgGaX9m4ydfhXGTnsfWrd/Debwa8UfqBRiz9yOluf+\nFfGzbkaya12JqyWiUvEdj6gIhqa5C16LELYNV7uS8TXXQ6By7dlcOj6WbIhOFAvlE88nkydgD76A\nieUXF3xsM/XgpdK42Tk/+s4vQOkWOh77FKCK6BOeTaPj53+J9m1fwujqa3H0nZ9zsVIiKhWDYqIi\ntIbMit1ON3St+B6os6RbTkNiyfkIv+yuCwWQyy0+Njbp+ni/KCp1YuApiMoWDIo1Edgl7gZS85j/\ns7rYKPEt2/tn/pwJLcHxSzYjcGgbIrvvynt+SY6i+z8/guiLd2Hogj/BkXd/A9DL7y9NRIXxnZ+o\nAMvQyppCV4yw7S5vc3TNb8E+tsf12GcAGE2kFyxSqxepTLaoKX3OwSdz/Ym7L8j/OFNvinxicsc2\nft2abcv2fuiazBkl/v0d+/HZ+3ZhTXdkznHxtTdhYvklaN/6ebQ8841cR4p5u8b66ACW3Xc9Avt/\ngcF3/R2GNvwV84iJqog/bUQFVGN6V9DS83YBWMzo6k1TY5/d7xYDwNHRybpt0VbMLjGQK7JLdF8A\nZQTyPq7Ru05QeURkZrd4TXcE/759P26+6BTc9uAefOnHe/Avj7+Oj16yak7Xk6kDMXj53yPZejra\ntn0ZK/7jN9FzRy86H/4ThPvugXPwSSy/52qYI/tw+Kr/i/jZH6rB346oubGsmCgPx9SrUn0vIgg7\nBobHkyUdl3XaML7ycoRf/iGO/8ZnAc1dLmwqk8XweMqzHszVVEw+sZYYhjW4C0MX/lnBx5Y7mIUa\nX8DSMTqZRm9PKzZvXIvbHtyDnrYAHt5zBO9e24Ub1p+y4HHpaA8O3fBj6GNvIrD/cQT7H0ew/zFE\nXs51kUmHunHo+vuQ7Di7mn8dIprCoJgoj5ZgZdMmZgvbpQfFQC6FIrT3Jwgc+CUmei51/fzDEymE\nbKPk6vpaSqazRaV+OIeegkAhsaJwPjGDYipk9mCX3p5WXLSqFQ/vOYJzl0exfd8QdvYPnbxTPEsm\ntASjZ/42Rs/8bUBlYQ3+CvaR5zG+6t3IhJdW469ARAuon6sfUZWZulbVHq2WobkKyMZWXYmMHUO4\n7wdlPb9SCkdH66vortiWcoGDTyCrO0gs6c3/OHadoCLMbs32/R378cjUDnH/8YmZVIr5xXeLEg3J\nrnWIn/M7CwbExRTyEZE3GBQTLSJWxV3iaRHHRRCu2xg7fSNCrz8ESY6V9fyJVAYn6qR3sVIKo0VO\n5XMObkVi6fqCVfzN1p+Y3AtaBnb2D+GOJ/bh1ktPw6ffvxabN66dyTHuOxz35HnWdEfmBNk7+4dw\n24N7TirkI6LylRUUi8gNIrJbRLIist6rRRHVmq4JIi47QpQjZBkzle2lyI19nkBo70Nlr2GoTnoX\nj06mi1qnNnEc9rE9SBTRn9ixuE9AxQlaOvoOx/GF686ZySGezjHOZBVuuqjHk+eZnbd8xxN7cduD\ne7B549q86RlE5E65V4BdAK4H8AsP1kLkG1Gncn2J89E0QchFMD659EKkoisR2fO9steQySociSd8\n343iRKK4XeLAoScBoGB/Yl0T2NwppiI5po4PbVh5UnDa29PqWUA8+5yb1i3Fd7f1Y9O6pQyIiSqk\nrKBYKbVHKdXn1WKI/EBEEK3Q9LpiuEqhEEF87U0IHNwKY3hv2WuYSGYw6OP84sl0BpOp4lqxOQe3\nImsEMdl1Xt7HBVhgRyWq1vfMzv4h3P/8AG7Z0IP7nx8oPl+ZiErCe4VE84RtA7pWu+ENjqnD1Ev/\n0YyvvRFKdET33O3JOkYTaQyNld4NoxpOTBS3SwwAgQNbkVh2EaDn/6BjMyimElWjMHM6h3jzxrX4\n6CWnzqRSMDAm8l7BK6+IPCIiuxb4dU0pTyQifygiO0Rkx+DgoPsVE1VYrIa7xNOiLiboZUJLML7q\nSoT3fA/IeBPMDo0ni+7wUC3ZbPEFdvr4IKyhlwumTgAc2kGlq0Z3mr7D8Tk5xNM5xl4U8okINBFO\ncCSaUvAnWil1pRdPpJT6FoBvAcD69ev9naxITSto+aNPb9gxcHw8WXJe74mzbsbSvT9BaN/DGDv9\nKk/WcnQ0CUPTfNOuLD6ZLvrfxTm4FQAKFtkxn5jc0DWBbepFp/K4sVB+cm9Pa8l5xbapoyNsQZ8K\ngjXBTDCcySoMxieLGoRD1Mhqf/Un8hE/7BIDuYtt0EUQOtFzGdLhpYi8+O+erUUphTdPJJBMZz07\nZzlOTBS/cx04uBVZM4zJznPzPo4BMbkVLqNLzZbt/fjqT/vmpELs7B/C137a52kf4qBlYGnUgW3o\nMHQNujZ3d1jXBN0xB+0hm7vG1NTKbcl2nYgcAPAbAP5TRH7izbKIqs8y/LMbCrgsuNN0xNd+EIH+\nx2Gc2O/ZWrJTgXGtW7UlUhmkMsUH587BrZhY9jZAy/9vySI7citsG64DyTXdEfy8bxCbf7QbO/tz\nk/A2/2g3Husb9KwPcdgxsCRqQyuiTiIWNLGsxXFV00DUCMrtPnGfUmqFUspWSi1RSv2mVwsjqja/\n7BJPC5i6q4K/E2tvAgBE9mzxdD2pTBYDIxNIlxCUeq2UXWJ97DCs4deL6k9sM5+YXHJ7VwfIpUHc\nds3ZAIDP3LcLn7lvFwDgtmvO9qTtWixgoivilBS024aO5S0BhN18KCeqc7wSEAEwNK2s26CVICKu\nCnkykeWY6LksFxRnvc0RTKazGBhJlLRb65VMVmEsWXzuZuBALp94YsUleR+nibgar000zU1v8Wm9\nPa24/vzlmExnMZnO4vrzl3sSELeHbLSH809wXIymCboijuvjieoVg2IiANGA+1ugleQ2UD9x9s0w\nxg4j+MajHq9oasd4OIHJdOWKixYST6RKKjx0Dm5Fxo4h2X5W3sdxl5jKFbJ0V5MogVwO8b3PHoRt\naLANDfc+e7DsdmudEduTMfWxgFnTnu1E1carATU9TcRVC7RqCFg6DK30H9PxlVciHej0tOButnQ2\ni8MjCSQqWHU/WyarSupNDOSK7BLLNgBa/l1g5hNTuUTcTaKcziEGgC9edw6+eN05ADCTY+xGNGAi\n4uH7WXvI4p0UahoMiqnphR2jqCKUWgnZLi5Iuon42hsRfONn0EcHvF8UcoHqwEiiKm2cjsQTSGeL\nT9nQ4wdhnnijyP7EvOBT+dwUxvYdjuNdazpncoinc4wvW9Ppqg+xqWtoC1olH5ePiGBJ1HH14Zyo\n3vgriZKoBvxWYDdfyDYwUkKB2Zbt/VjTHcGFZ30Qrc/ejshL38NjXb+LvsPxBXueliPXrm0SnZHy\nWlPlMzSWxEQJucQAENz/OABgYsXb8z5ORGD7oC811b/pSZSl5Nt71YN4Wke4uC4TpdI1QVfUxsBI\nouTe6UT1hFcDamoh2/B9+6FSxz6v6Y7gtgf34OmRGMZXvB32C/+Ozz2w27MWT/MppXDkRALHRic9\nv2COJ9MYGi99Ol9w78NIRVYg1bYm7+NsQ/NlLjnVp3IK7soVccyKtpR0TB3tYW93oYn8xt/RAFGF\n+X2XeFopu7DTY2Bve3APHtDfjdDEQXzjbcOeVLTnMzKRwqGRhGct21KZLAbjkyUfJ+kJBA78F8ZX\nXQkUCHiZT0xeqlUHG0PT0B6qfMAadbzNVybyGwbF1LRsU6+bfNJSd6B6e1qxad1S/I++VRjTYzh/\n8IcVWtlck6kMDg5PlJ1nrJTCkfikq2EhzoGt0NITGF/17sKPrZPXn+qDZWiwa/A91RGxqlYX0RFm\n4R01LgbF1LTqZZcYyF1srRJyX3f2D+H+5wdw44bTcXfmcgRf/wmMkX2VW+AsmazC4ZEEjo8lXadT\nHB1NYnKRzhZbtvefVJm/s39oZixu8I1HkDWCuc4TeYgIHLZjI49Ve7c47Biu+pm7JSLoitiuW9AR\n+RmvCNSUTN1/wzoKidjFBfE7+4dw24N7sHnjWnz0klOx9N0fRxqC5C+/WeEVzjU8nnTVnSKeSCGe\nWLywcDpnejownv77rumOAEohtO8RTPRcCmU4eZ/HYj4xVUA5Y59LlUubqP6ADUPXmF9MDYlBMTUl\nv/YlzqfY1mx9h+PYvHHtTA7x2jPOwMFTrkZP/73QEscrucSTJFIZHB5JYP/xcYyMpxZNh8hmFcYm\n0xiMT+LoaP7Cutk503c8sXfmA0BvTyusYy/CGD2E8ZVXFlwb84mpEnRNqva91R62XI2C90LEMau6\nQ01UDQyKqeloIq56itaaoWtF5fLddFHPSUV1+iUfh5VNILrrO5VaXl6pTBbHxibRf3wcR+K5oR+J\nVAZDY0kcHJ7AvmNjePNEouipddM509/d1o9N65bO/H2D+x4GAIyvuqLgOZg6QZUSrsL7S9g2atrt\nAsjlFzONghoJrwrUdCI+H9aRj9uLbap9DcZXXo7oC3dA0gmPV1U8pRRGE2kcGp7AoeEJDI0vnjuc\nz3TO9C0benD/8wMzqRTBfY8g0dWLTLCz4DkcgzvFVBnljH0uhq4J2sPVT5uYj2kU1GgYFFPTqacC\nu/lClvt8xeHzboUxcRThvns8XlVlLFZQ99Wf9s3JmZ5OpXjxlVdgv/kcxk8tnDphGVrdfjAi/5MK\n341qD9s1S5uYj2kU1EgYFFNTCdsGDJ8P68innHzFxPKLMdn5VsSe+1dAedNLuJIWK6gTYE7O9HSO\nMV5+GAJVVCs25hNTpbUGrYqMRg7Zhu+KhJlGQY2ifqMDIhdiwfrdJZ5WagrFzI6rCIZ7b4U1/BoO\nP/3DmRZmfrVYQd2fvWfNSTnTvT2tuEx7FunwMiTbzyp4bvZZpUrTNEGbx6kFuibo8EHaxHxMo6BG\nwaCYmkbINmA3QB5pqfmKs3dcx06/CmOBZbCf/qdFxz4X6gNcTYsV1J0kM4lA/+MYK2KKHcCgmKoj\nbBuejl5uC9Wu20QhTKOgRsCgmJpGSwPsEgO5fMVSqs7n7Lg+uR+3T7wHF2APNlh7F3x83j7AVbZY\nQd18gYNPQkuPF9WKzdQ13wYW1HjaQ7YnfYuDluH7EctMo6B6x6CYmkKj7BJPK7WIZ/aOa/Lcm5Gx\nY4g99y+LPnaxPsDVNH8IyfSaFgqMg/seQdYIILHikoLn9XLnjqgQy9DQUmZxryaCjjpIT2AaBdU7\nBsXUFBpll3iaY+owSygYnL3j+oNdI3jllBsQev2hRUc/F522UEHzh5BMB+t9h+NzH6gUgvsexsSK\ndxScYgewyI6qryVolvTzOl972KqbAmGmUVA9q4+fMqIyNNou8bRiK9AX2nH9o1cvQlZ0xJ7/9qLH\nFJO2UEkLDSHp7WnFTRf1zPmaefwlmPEDGD+1cNcJgPnEVH0i7gvkIo7p+7SJ+ZhGQfWKQTE1vEbb\nJZ5WbBeKhXZcP3b1xfhV23sRefFu6PGDcx5fStqCH4T2PQIAGF9ZeIqdZTCfmGojYOklt1JrD9no\njPiv20QhTKOgesWgmBpauEF3iYFcwZjbsc+9Pa1oe99fQ1QWbdu/Ouf/FZ224BPBfQ/jUHAtdhyb\nexFeqGMGUyeoltpCxe2gaiLojjl13UKSaRRUjxgUU0Or54tKMcqZmpWOrsDIW38P4Zf+A+axPTNf\nLzZtwQ+0iWOwDz+L4RWXFdUxg0V2VEuGrqEraiOYZzKlqWtY1hJoiICSaRRUbxgUU8Nq5F3iaSHL\nKOuiM3zBHyNrR9H+5Bc9XFX1hPbmpthF111dsGOGiMBp8O8H8r+gZaA75qCnLYiOyNwAOWDpWN4S\ngGU0xqWZaRRUbxrjJ49oAY2+SwzkpmYFbfeBXtZpxfAFH0fwjUfhHHjCw5VVR2TP3Ui2nI5k57kF\nO2bYhgaN+cTkE7omiDrmTIC8JOpgaSzQcN+jTKOgesKgmBpSM+wST4vY5QX/J879CFLh5Wh7zf1u\nswAAElBJREFU8guAynq0qsozj+2Bc3gH4md/CBAp2DGDXSfIr3SttIE89aYj7N9JfESzMSimhiMi\naAk2zy27gKXD0Nz/KCvDwdDb/gLOkecRevUBD1dWWdHddyKr24ifeUNRHTNYZEdUG4au1WUXDWo+\nDIqp4bQEzIbJyStWse3ZFjN6xvWYbD8Lbdu+DGQmPVpV5UhqHJG+ezB2+lXIOm0FO2aICByzub4n\niPwkaBmIlTnZj6jSeJWghmLqWsP2Jc6nmP6nW7b3n5RSMNO2TNNx/OLPwjzRj+iu71ZqmZ4Jv3I/\ntGQc8bM/DKBwxwzH1Bat9iei6mgLWbB5x4Z8jEExNZSOsN2UwY9laAUvNmu6I3nblk2ccinGV7wD\nrTu+Dpk8UfE1lyOy+04kW1cjsfSioh7P1Ami2hMRdEVs5heTbzEopoYRdoym7kNbqGfxdErBom3L\nRHD84s9CTwyh5dlvVmHF7liDu+Ac2YkTZ38YKPIDEIvsiPzB1DXXI6+JKo1BMTUEXRO0h5r7jTZc\nRM/iQm3Lkp3nIr76OsSe/zaM4dcruVzXcgV2DkbXfKCox2siDIqJfCRkM7+Y/IlBMTWE1hBb/mhF\ntHUq1LYMAI5f8tdQho2un/0pkM1UarmuSHIM4Zfvw9hbNiLrtBR1DANiIv9hfjH5EYNiqnuOqSPq\ncNcBAKKBxYPiYtqWAUAm1I1j7/g8nMM7EHvuW5VecknCr/wQWmo0lzpRJOYTE/kP84vJjxgUU10T\nEeanzWIb+qI7o4Xals02esZ1GDvtfWh76u9gHuur6JpLEd19J5JtZ2Kye33RxzgW3+aI/MjUNSyJ\nOk1ZHE3+xKsF1bVYE/YkLiS6SK5eobZlc4hg8NIvI2uF0fWzTwKZVCWWWhLryAuwB18oqcBO16Rp\nJhsS1SPH1LEk2pxdg8h/GE1Q3XJMHa1N2JO4kJCle3JLMhvswNF3fRn24AtoefZ2D1ZWnujuO5E1\nHIyuub7oY5g6QeR/QcvgxDvyBQbFVJd4221xIoKIRznWY6dfhdHV16J1x9dhDe7y5JxuSDI+VWB3\nDbJ2rOjjnCZu0UdUT8K2gXamwlGNMSimuqOJoCvKAo18omWOfZ7t6Ds/j4zThs6ffbJmI6AjffdA\nS4+XVGAHAA5TJ4jqRixgoiVo1XoZ1MQYFFPd6YrazBMtwNC1gu3ZipV1WjF42VdgH9uD1qf/wZNz\nlkIfH0TrU19BYumFmFzSW/RxhqYx35yozrSFLM/udBGVilcMqivtIRtBy7td0EbmZZu6iVVXIn7m\njWh59psI9P/cs/MWo/0Xfw0tNY7Bd32l6AI7gF0niOpVZ8RG2MO7XUTF4lWD6kbYMRBjYV3RApYO\nU/fuR/zoO/4GybYzseShP4D95nOenTef4Gs/Rvi1BzF04Z8i1ba6tGP54YmobnVFHLSFmEpB1cWg\nmOqCY+roZBFGybzcLVZWBIevvhOZQDu6H7yl4mOgtcQQOh7/DCY7zsZw738v6VgRQZCdJ4jqWkvQ\nQnfMKTi+nsgrDIrJ92xTZ6cJl8KO4em/Wya0BIc33QUAWPrAh6CPHfHs3PO1P/E30CeHMHj51wC9\ntODeMTVoLMQkqntBy8CyloCnd72IFsPvMvK1sG1gWcxhpwmXdE0Q9qjgblqq5XQc3vgd6OOD6H7w\nw5DkyRPxyhV441FEXvo+hns/hmTnOSUfz9QJosZhGRqWtQQWndZJ5BUGxeRbbSELXdwhLls04H2A\nOLmkF2++93/DOt6H7h//vqet2iQZR+fP/xLJ1tUYuvCTrs4RZH9iooaia4KlMQexRSZ2EnmBQTH5\njiaCJVGH/So9Yht6RXZYJlZehsHLv4rAwSfQ9cgngGzGk/O2PflF6KMDGLz87wG99Dxyy9B4q5Wo\nAYkI2sM2lrdy15gqg/cYyVdMXWMf4gqIBUwkUt4ErbONrvkA9LFBtD/5eejjRzF4xdeQjva4Pp9z\ncCtiu76D4XV/gMnu9a7OwdQJosZmGzqWtQQQT6QwNJZCOput9ZKoQXA7hXwjbOcKKhgQey9kGxXb\nPR3pvRVHLv8a7MFfYcXdVyCy6zuAUqWdJJNC7Jnb0f3ALUhFV2HobZ9yvR6mThA1h4hjYkVrALGA\nyTQ78gSDYqq5oGVgeWsAXVEW1FVStFK5eCIYXXsj9n/wUSS6L0Dn459G9wM3Q48fLOpw+83nsPz7\n70f7ti9hoPPtOHT9PVBmEACws38IW7b3F70UQ9N4W5WoiWjaVEpFSwDRgMn2bVQWBsVUM46ZuwXW\nHXO4O1wFUceo6IeOTGQ5Dm+6G4OXfhHOwA6csuUKRF7cAqiFb21Kcgzt/7UZy+65GnriOJ5c/494\n/5v/DTuO5fKId/YP4bYH92BNd6ToNQS4S0zUlCxDQ0fYxsr2IDoiNmx+OCYXRJV6m9MD69evVzt2\n7Kj68w7GJxFPpKr+vDSXY+poCZrM/ayBobEkhsaTFX8eY+QNdD76Zwgc2galmUgHu5AJL0U61I10\neCmyTisiu++CMXoIJ875HRzf8FdQdnQmEN60binuf34AmzeuRW9Pa9HP2x1z+H1FRACARCqDeCKN\n8WQamWz1Yx2aq6ctCKNGRdAi8oxSqmChCq8eVHEigoCpI2jrCFmV3a2k/KIBE8MTKVT6w3A6thID\n134foVfuh3VsD4yxARijA7COvYjgGz+Dlp5AsvUMHLr+PkwuvXDmuN6eVmxatxTf3daPWzb0lBQQ\na1PfZ0REQG4DJpdOZWMynUEimcVEKoOJVKbi74FUnxgUk+dEBIYmsE0NQctA0NQ5Xcwnpod5VOWO\niWgYO+NajOHauV9XCpKMQ1lhQObuGuzsH8L9zw/glg09uP/5AZx3SkvRgXHA0llsQ0QLsg0dtqEj\nBhNKKSRSWUymM0hmskims0hlFANlKi8oFpGvALgaQBLAawA+qpQa9mJh5H+6JtA1gaFpMHWBZWi5\nX7rG4MTHYgGztmlEIlB29KQvT6dOTKdMnHdKy5w/T9uyvR9ruiNzvrazfwgHhibwiSvPqMpfgYjq\nl4ggYOkn1SDkguMs0hmFdDaLTFYhnVUz/2XQ3PjK3Sl+GMCnlVJpEflbAJ8G8JflL6sydE2atqn/\ndIw6O1iVqa8LZOq/v36MJif/V5PpIDj3Xwa+9ckycjv448l0rZcyR9/h+JwAuLenFZs3rkXf4fic\nAHhNd2ROsDwdTH/z5vNrtXQiagDTGzuLyWQVsmrqVxZzfq+goFTuawq5rpTTQfT0n3O/V7N+/2vz\nA27G37XhWaGdiFwH4ANKqQ8VemytCu2IKCeRyuDQ8EStl+Ha/IK8z197Dq5et6zWyyIiIh8qttDO\ny23T3wPwUJ4F/aGI7BCRHYODgx4+LRGVyjH1um5ZNLsgb9O6pXjH6o5aL4mIiOpcwaBYRB4RkV0L\n/Lpm1mM+CyAN4K7FzqOU+pZSar1San1nZ6c3qyci12KVGuZRBfML8l44MFLrJRERUZ0rmFOslLoy\n3/8XkY8A2AjgCsUsdKK6EbJ0mLqGVGbh4Rp+Nb8gb/3KNnzye8/h9pt7cfHp3DEmIiJ3ykqfEJH3\nAvgUgE1KqXFvlkRE1SAilRv9XEHzC/LevroDt9/cy91iIiIqS1mFdiLyKgAbwLGpL21TSt1a6DgW\n2hH5g1IK+49PIJ2tr93i2Za1BKYa9BMREZ2sKhPtlFJvKed4IqotEUEsYOLY2GStl+KKZWgMiImI\nyBPN2bSXiGZEnPodvR1x6i/9g4iI/IlBMVGT0zTxTSeKLdv7sbN/aM7XdvYPYcv2/pMeKyKI2JxU\nT0RE3mBQTESIOiY0H0wonJ5WNx0YT3eaWNMdOemxIVuHVqc73ERE5D/cZiEiaFquE8XweLKm65ge\n7Tx7Wt3sThOzRZk6QUREHuJOMREByA3z8MNu8fxpdQsFxKbOAjsiIvIWg2IiAgDomiDi1P7m0fxp\ndfNzjAHuEhMRkfcYFBPRjFjAhNRwt3j2tLqPXnLqTCrF7MBYRBD2QfBORESNhUExEc0wdA3hGnZ0\nmD+tbjrHuO9wfOYxIUuv2xZyRETkX9xuIaI5WoImRifTKGfapVs3XdRz0td6e1rn5BWzNzEREVUC\nd4qJaA6zxrvF+Zi6hoDFAjsiIvIeg2IiOklr0B+dKObzQyEgERE1JgbFRHQSQ9d8M+Vumoj4dgeb\niIjqH4NiIlpQLGDC0PzzFhG0dBi6f9ZDRESNhVcYIlqQpglaQv7ZLWbqBBERVRKDYiJaVNQxYRm1\nf5twTB1Bi0ExERFVTu2vdkTka+0hu9ZLQFvIqvUSiIiowTEoJqK8AlZtd2nDjgHHZBs2IiKqLAbF\nRFRQrXZqRQRtQe4SExFR5TEoJqKCLEOrySS5loDJjhNERFQVvNoQUVGqPdDD0PzXK5mIiBoXg2Ii\nKoqha2itYipDa8iEpvlvqh4RETUmBsVEVLRY0ESoClPlapWuQUREzYtBMRGVpDNsw6xwnm9HuPZt\n4IiIqLkwKCaikmiaoCtqQyqUXxyy2YKNiIiqj0ExEZXMNnR0hL3PL9ZEOKiDiIhqgkExEbkScUxP\n8341EXTHnIqnZhARES2EVx8icq0jbMH2INVhOiBm2gQREdUKg2Iick1EsCRiQy+jdRoDYiIi8gMG\nxURUFkPXsCTqLu2BATEREfkFg2IiKptj6ljRGkB7yC566h0DYiIi8pPKd+EnoqYgIlPDPXQcH09i\nNJFe9LGmrqEzYjMgJiIi32BQTESeMnQNXREHUSeDo6OTAHIT6mxDh21osHSN45uJiMh3GBQTUUXk\nUiqCtV4GERFRUZhTTERERERNj0ExERERETU9BsVERERE1PQYFBMRERFR02NQTERERERNj0ExERER\nETU9BsVERERE1PQYFBMRERFR02NQTERERERNj0ExERERETU9BsVERERE1PQYFBMRERFR02NQTERE\nRERNj0ExERERETU9UUpV/0lFBgG8UfUnbiwdAI7WehF0Er4u/sPXxH/4mvgTXxf/4WvijZVKqc5C\nD6pJUEzlE5EdSqn1tV4HzcXXxX/4mvgPXxN/4uviP3xNqovpE0RERETU9BgUExEREVHTY1Bcv75V\n6wXQgvi6+A9fE//ha+JPfF38h69JFTGnmIiIiIiaHneKiYiIiKjpMSiuYyLyFRF5SUReEJH7RKSl\n1mtqViLyXhHpE5FXReSvar0eAkTkFBF5TEReFJHdIvKJWq+JckREF5GdIvJgrddCgIi0iMgPpq4n\ne0TkN2q9JgJE5E+n3rt2icjdIuLUek2NjkFxfXsYwDlKqbcCeBnAp2u8nqYkIjqAbwJ4H4CzAHxQ\nRM6q7aoIQBrAnyulzgKwAcAf8XXxjU8A2FPrRdCMfwTw/5RSZwJYB742NSciywF8HMB6pdQ5AHQA\nN9V2VY2PQXEdU0r9VCmVnvrjNgArarmeJnYRgFeVUq8rpZIAtgC4psZranpKqQGl1LNTv48jd6Ff\nXttVkYisAHAVgG/Xei0EiEgMwDsB/B8AUEollVLDtV0VTTEABETEABAEcKjG62l4DIobx+8BeKjW\ni2hSywHsn/XnA2Dw5SsisgpAL4CnarsSAvB1AJ8CkK31QggAcCqAQQB3TKW0fFtEQrVeVLNTSh0E\n8PcA+gEMABhRSv20tqtqfAyKfU5EHpnKJ5r/65pZj/kscreK76rdSon8SUTCAO4B8Eml1Ilar6eZ\nichGAEeUUs/Uei00wwBwPoB/Vkr1AhgDwLqIGhORVuTuOJ4KYBmAkIh8uLaranxGrRdA+Smlrsz3\n/0XkIwA2ArhCsb9erRwEcMqsP6+Y+hrVmIiYyAXEdyml7q31egiXANgkIu8H4ACIisidSile7Gvn\nAIADSqnpuyg/AINiP7gSwF6l1CAAiMi9AC4GcGdNV9XguFNcx0TkvcjdhtyklBqv9Xqa2NMAVovI\nqSJiIVcMcX+N19T0RESQy5Pco5T6Wq3XQ4BS6tNKqRVKqVXI/Zw8yoC4tpRShwHsF5E1U1+6AsCL\nNVwS5fQD2CAiwan3sivAAsiK405xfbsdgA3g4dzPDLYppW6t7ZKaj1IqLSJ/DOAnyFUI/5tSaneN\nl0W5XclbAPxKRJ6b+tpnlFI/ruGaiPzoTwDcNfWh/nUAH63xepqeUuopEfkBgGeRS4/cCU63qzhO\ntCMiIiKipsf0CSIiIiJqegyKiYiIiKjpMSgmIiIioqbHoJiIiIiImh6DYiIiIiJqegyKiYiIiKjp\nMSgmIiIioqbHoJiIiIiImt7/B0La3GoHpmeiAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "f = gpflow.models.GPR(X, Y, gpflow.kernels.RBF(1))\n", "f.compile()\n", "gpflow.train.ScipyOptimizer().minimize(f)\n", "full_lml = plot_model(f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Upper bounds for sparse variational models\n", "As a first investigation, we compute the upper bound for models trained using the sparse variational GP approximation." ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 49.220787\n", " Number of iterations: 25\n", " Number of functions evaluations: 36\n", "4 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 40.211568\n", " Number of iterations: 52\n", " Number of functions evaluations: 79\n", "5 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 37.583478\n", " Number of iterations: 35\n", " Number of functions evaluations: 43\n", "6 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 30.092951\n", " Number of iterations: 32\n", " Number of functions evaluations: 49\n", "7 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 26.316172\n", " Number of iterations: 33\n", " Number of functions evaluations: 45\n", "8 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 24.957100\n", " Number of iterations: 30\n", " Number of functions evaluations: 36\n", "9 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 24.310472\n", " Number of iterations: 35\n", " Number of functions evaluations: 41\n", "10 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 24.069186\n", " Number of iterations: 45\n", " Number of functions evaluations: 63\n", "11 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 24.014035\n", " Number of iterations: 64\n", " Number of functions evaluations: 81\n", "12 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.981403\n", " Number of iterations: 58\n", " Number of functions evaluations: 72\n", "13 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.979629\n", " Number of iterations: 76\n", " Number of functions evaluations: 92\n", "14 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.979570\n", " Number of iterations: 67\n", " Number of functions evaluations: 76\n", "15 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.966980\n", " Number of iterations: 175\n", " Number of functions evaluations: 209\n", "16 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.966979\n", " Number of iterations: 236\n", " Number of functions evaluations: 278\n", "17 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.966959\n", " Number of iterations: 163\n", " Number of functions evaluations: 193\n", "18 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 23.966942\n", " Number of iterations: 127\n", " Number of functions evaluations: 158\n", "19 " ] } ], "source": [ "Ms = np.arange(4, 20, 1)\n", "vfe_lml = []\n", "vupper_lml = []\n", "vfe_hyps = []\n", "for M in Ms:\n", " Zinit = X[:M, :].copy()\n", " vfe = gpflow.models.SGPR(X, Y, gpflow.kernels.RBF(1), Zinit)\n", " vfe.compile()\n", " gpflow.train.ScipyOptimizer().minimize(vfe, disp=False)\n", " \n", " vfe_lml.append(vfe.compute_log_likelihood())\n", " vupper_lml.append(vfe.compute_upper_bound())\n", " vfe_hyps.append({p.full_name:p.read_value() for p in vfe.trainable_parameters})\n", " print(\"%i\" % M, end=\" \")\n", "vfe_hyps = pd.DataFrame(vfe_hyps)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAGDCAYAAAD+sAySAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd8VFX+//HXSQ8EQu8lFOlFOhZAxAKCsJa1rHV1Vey6\nll3Xr6vu6qqruzb0Z19dRcUushaUSFMQadJLAqH30ENCkjm/P84NTGII6TczeT8fj3nMzL137nxm\nEsJ7zpxirLWIiIiIiEj5ifC7ABERERGRcKOQLSIiIiJSzhSyRURERETKmUK2iIiIiEg5U8gWERER\nESlnCtkiIiIiIuVMIVtEKp0x5iFjzDtVoI4kY4w1xkSV8HHGGPMfY8xuY8yciqqvvBhj0owxZxTj\nuFK9H+XBGDPIGLOygs79pjHmkTKe4y/GmNeK2H+1MWZmWZ5DRMKLQraIAMcOYsaY07zg9WmB7T29\n7VODtlljTPtKKNdvpwJnAi2stf39LsZv5fGhyVo7w1rbsbxqKm/W2n9Ya/8A5fNhxBhzqjHmR2PM\nXmNMujHmB2NMv6D9TY0xrxpjNhtjDhhj1ngfFjoVqOGAd0kzxvw56PHWGHPQ27fJGPNvY0xkWd4D\nESkZhWwRKY4dwEnGmPpB264CVvlUj99aA2nW2oMlfaAfrcR+81r+9f+NxxhTG5gEPA/UA5oDDwNZ\n3v76wI9ADWAQUAvoDUzDfbgLVsdamwBcCvzVGDM8aF9Pb98Q4GLgmop6TSLya/qjJyLFcRj4DLgE\nwGsRuxgYX4ZzxhljJhhj9htj5htjeubtMMZ0NsZMNcbsMcYsNcaMDto31Rjzh6D7+b6m91rwxhpj\nVnuPf8EYY/LqNsY8ZYzZaYxZA4wMLsg71xqvprXGmMsKFm2MuRZ4Dfeh44Ax5mFv+3XGmBSvVXKi\nMaZZgZpuNsasBlYXcs68VsnfG2M2eN1Qxhpj+hljFnmvY1zQ8RHGmP8zxqwzxmw3xvzXGJMYtP8K\nb98uY8z9BZ4rwhjzZ2NMqrf/A2NMvcJ+QMV8P4YDfwEu9t6PX4J+To8aY34AMoC23utb7p1vjTHm\nhqDznGaM2Rh0P80Yc7f3+vd6vytxQftHGWMWeu/Nj8aYHkH7enm/U/uNMROAI48rpP51xpg+3u3L\nvJ9DV+/+tcaYz7zbwa31073rPd5rPinofE95P7+1xpgRx3jaDgDW2vestbnW2kPW2snW2kXe/juB\nfcAV1tpU6+yx1v7HWvt8YSe01s4ClgLdCtmXAvwAnHis90FEyp9CtogU13+BK73bZwNLgM1lON8Y\n4ENcS967wGfGmGhjTDTwBTAZaATcCow3xpSkK8EooB/QA7jIqxfgOm9fL6AvcGHeA4wxNYHngBHW\n2lrAycDCgie21r4OjAVmWWsTrLUPGmNOBx7znqspsA54v8BDfwMMALoUUfcA4ATcB5hngPuBM4Cu\nwEXGmCHecVd7l6FAWyABGOe9ji7A/wOuAJoB9YEWQc9xq1fLEG//buCFgoWU4P34GvgHMMF7P3oG\n7b4CuB7XErsO2I57/2sDvweeNsb0LuL9uAgYDrTB/Syv9mrrBbwB3OC9vpeBicaYWGNMDO4D4du4\n360PgQuKeI5pwGne7SHAGmBw0P1phTwmb38d7zXP8u4PAFYCDYB/Aq/nfcArYBWQa4x5yxgzwhhT\nt8D+M4BPrbWBIuo+wjin4H5PFhSyvxOuRTylOOcTkfKhkC0ixWKt/RGo54XdK3GhuyzmWWs/stZm\nA//GtTYO9C4JwOPW2sPW2mTcV+uXluDcj3stf+uB7znagncR8Iy1doO1Nh0XjIMFgG7GmHhr7RZr\n7dJiPt9lwBvW2vnW2izgPlxLd1LQMY9Za9OttYeKOM/frbWZ1trJwEHgPWvtdmvtJmAG7sNB3vP9\n21q7xlp7wHu+S4zrinIhMMlaO92r5QHvdeUZC9xvrd3o7X8IuNAU3o2ltO9HnjettUuttTnW2mxr\n7f+CWman4T5IDSri8c9Zazd7P6svOPpzvB542Vr7k9cS/Bauq0Xe70807uecba39CPi5iOeYhgvT\neLU8FnT/WCH7WNZZa1+11uYCb+E+cDUueJC1dh+uX78FXgV2eN9+5B3bANiad7wxZrTXYr/fGDO5\nwOl2Aum4b1f+bK2dErRvvjHmILAcmAq8WILXIiJlpJAtIiXxNnALrgX10+Mcezwb8m54LXYbcS2r\nzYANBVrx1uH6rRbX1qDbGbjQTt65C5w3r4aDuBbkscAWY8z/vBbA4mhW4FwHgF0Fat5Q8EGF2BZ0\n+1Ah94Nfx7qgfeuAKFygy/cavde1K+jY1sCnXmjbgwtguRQIg2V8P/Lke81eq+1s47rU7AHOwQXK\nYznWz7E1cFfea/DO1ZKjvz+brLU26LHB71VB04BBxpimQCTwAXCK9wEpkUJa74tTr7U2w7uZUNiB\n1trl1tqrrbUtcF08muG+vQD382oadOxEa20dXDeSmAKnamCtrWut7Wytfa7Avt7e81+Ma2WvWYLX\nIiJlpJAtIiXxNnAT8GVQiCitlnk3jBsU1wLX/WQz0NLkHyjXCtjk3T6IGxCWp0kJnnNL8PN65z3C\nWvuNtfZMXMBZgWtlLI7NuOAHHOlqUT+oZnCtluUl3/PhXkcOLpTne43GmBpeLXk24LqA1Am6xHmt\n5fmU4P041ms7st0YEwt8DDwFNPZC45dAYd0pjmcD8GiB11DDWvse7vU3L9BNo1XhpznSXzkD141m\nutfKvBXXWj7zGF02yvNnibV2BfAmR/tTTwF+Y8phsKj3rcEHwCzgr2U9n4gUn0K2iASLNsbEBV3y\ndSGw1q7FfYV+f+EPByCmwDmONW1YH2PM+d5z3IH7un828BMu9Nzr9dE+DTiXo32cFwLnG2NqGDdd\n4LUleH0fALcZY1p4/WCDpzxrbIwZ4wXkLOAA+btZFOU94PfGmBO9MPkP4CdrbVoJaiuJ94A7jTFt\njDEJHO0TnQN8BIwyboq4GOBv5P9b/xLwqDGmNYAxpqExZkzBJyjh+7ENSDpOKIwBYnEz1eR4gwLP\nKsFrDvYqMNYYM8Drj1zTGDPSGFMLFyZzcD/naGPM+cDxplmchvuGJq9ryNQC9wvagXsv2pameGNM\nJ2PMXcaYFt79lrjuULO9Q/4N1AXeNsa0815jLco2cPFx4DpjTEk+lIpIGShki0iwL3HdEvIuDxU8\nwFo701pb1IDHpQXO8ftjHPc57mvs3bgBcud7fWgP40L1CFx/0xeBK73WPoCncbOdbMP1ey3JDCev\nAt8AvwDzgU+C9kUAf8S1EqfjPkzcWJyTWmu/w/V9/hjXktoObyaWCvIG7luF6cBaIBPXEovXb/pm\n3GDSLbj3d2PQY58FJgKTjTH7ccFuQCHPUZL340PvepcxZn5hB1hr9wO34T7o7AZ+59VRYtbaubhB\nrOO8c6XgDYr0fn/O9+6n437HPinsPEGm4QZnTj/G/YLPnwE8CvzgdVcZWMKXsB/3nv/k9ZmejRtI\nfJd3/p24vuWZwEzv+IVeTcX6nSyk5sW413NPaR4vIiVn8ndbExERERGRslJLtoiIiIhIOVPIFhER\nEREpZwrZIiIiIiLlTCFbRERERKScKWSLiIiIiJSzwpbRDTkNGjSwSUlJfpchIiIiImFu3rx5O621\nDY93XFiE7KSkJObOnet3GSIiIiIS5owx64pznLqLiIiIiIiUM4VsEREREZFyppAtIiIiIlLOwqJP\ntoiIiIiUr+zsbDZu3EhmZqbfpfgiLi6OFi1aEB0dXarHK2SLiIiIyK9s3LiRWrVqkZSUhDHG73Iq\nlbWWXbt2sXHjRtq0aVOqc6i7iIiIiIj8SmZmJvXr1692ARvAGEP9+vXL1IqvkC0iIiIihaqOATtP\nWV+7QraIiIiIVEkJCQl+l1BqCtkiIiIiUq3l5uaW+zkVskVERESkSrPWcs8999CtWze6d+/OhAkT\nALj55puZOHEiAOeddx7XXHMNAG+88Qb3338/AO+88w79+/fnxBNP5IYbbjgSqBMSErjrrrvo2bMn\ns2bNKveaNbuIiIiIiBTp4S+WsmzzvnI9Z5dmtXnw3K7FOvaTTz5h4cKF/PLLL+zcuZN+/foxePBg\nBg0axIwZMxg9ejSbNm1iy5YtAMyYMYNLLrmE5cuXM2HCBH744Qeio6O56aabGD9+PFdeeSUHDx5k\nwIAB/Otf/yrX15VHIVukKIczYOPPYHMhIsq7RENEpLsdGe1ti/S2522LDDrW21+NB4+IiIiUxcyZ\nM7n00kuJjIykcePGDBkyhJ9//plBgwbxzDPPsGzZMrp06cLu3bvZsmULs2bN4rnnnuOtt95i3rx5\n9OvXD4BDhw7RqFEjACIjI7ngggsqrGaFbJGCcrIgZQos+RhWfgXZB8vnvEdCetClsJAeEQWRJTy2\ndnPoOAKa9FCYFxGRclfcFufK1rx5c/bs2cPXX3/N4MGDSU9P54MPPiAhIYFatWphreWqq67iscce\n+9Vj4+LiiIyMrLDaFLJFAHKzYc00WPoJLJ8EWXshvi50vxA6jYK42u6YQE7+S242BHK9+3n7c4OO\nDdqfG7Q/EHSu3JwiHu9dcjKLPvbANpj6GCS2hE4j3aXVyS6Ai4iIhLhBgwbx8ssvc9VVV5Gens70\n6dN58sknARg4cCDPPPMMycnJ7Nq1iwsvvJALL7wQgGHDhjFmzBjuvPNOGjVqRHp6Ovv376d169YV\nXrP+B5bqK5ALaTNdsF42EQ6lQ2xtF6q7nQ9tT3Otx6HgwA5Y9TWs+B/MexN+egni6kCH4S5wtx8G\nMTX9rlJERKRUzjvvPGbNmkXPnj0xxvDPf/6TJk2aAC6AT548mfbt29O6dWvS09MZNGgQAF26dOGR\nRx7hrLPOIhAIEB0dzQsvvFApIdtYayv8SSpa37597dy5c/0uQ0JBIAAbfvKC9eeuBTi6putq0e18\naDcMouP8rrJsDh+E1GQXuFd+BZl7ICrOfWjoNBI6jICEhn5XKSIiVdzy5cvp3Lmz32X4qrD3wBgz\nz1rb93iPVUu2hD9rYfN8WPIJLP0U9m1yofOEM6HbBXDC2RBTw+8qy09MTeh8rrvk5sD6WS5wr/if\na+3GQKuB0PEcF7rrt/O7YhERkbCjkC3hyVrYtsQL1p/A7jQ3WLD9MDjjIddyHVvL5yIrQWQUtBnk\nLsMfg62LvRbu/8G3D7hLw85eP+5zoFlvDZwUEREpBwrZEl52rHTBesnHsGs1mEhoOwQG3Q2dR7nB\njNWVMdC0h7sMvQ92r4OVX7rQPfNpmPEU1GrmwnankdD6VIiK8btqERGRkKSQLaEvfc3RriDblgAG\nkk6FgTdClzFQs4HfFVZNdVu792jgjZCRDqu+gRWTYMF4+Pk1iE10XWo6jYT2Z7gZVkRERKRYFLIl\nNO3Z4EL10k9g8wK3reUAGP6EC9a1m/pbX6ipUQ9OvNRdsg/BmqkucK/8CpZ8BJEx0GaIa+XueA7U\nauJ3xSIiIlWaQraEjv1b3YwgSz52M4QAND0Rzvw7dD0P6rT0t75wER3v+qx3HOGmOdzwkzdwchJM\nutNdWvRzLdwdR0LDDn5XLCIiUuUoZEvVdnAXLP/cdQdJmwlYaNQVTn/ABWvNjFGxIiKh9cnuctYj\nsH350cD93UPuUv8Eb+DkKGjeByIi/K5aRETEdwrZUvUc2uNC3JJPXLcFm+uC3JA/ubmsG3b0u8Lq\nyRho3MVdhtwDeze67iQrJsGscfDDM5DQ2LWAdxoFbQZDVKzfVYuIiBSLtRZrLRHl1FikkC1VQ9Z+\nr//vJ5DynVs2vE5rOOU2N5d1426aWq6qSWwB/a9zl0N7YPW3LnAv/sitOhmT4AZOdhzpruPr+F2x\niIiEmLS0NEaNGsWSJUsAeOqppzhw4ABTp06lZ8+eTJs2jZycHN544w369+/PQw89RGpqKikpKezc\nuZN7772X6667DoAnn3ySDz74gKysLM477zwefvhh0tLSOPvssxkwYADz5s3jyy+/LLfVIBWyxR85\nh2HnSjdv88qvYPVkyMl0U8gNuAG6ng/NNWdzyIivAz1+6y7ZmbB2upuLe8WXboBqRDT0uRqG/sUN\nshQRkdDy1Z/d/9nlqUl3GPF4qR+ekZHBwoULmT59Otdcc82RIL5o0SJmz57NwYMH6dWrFyNHjmTJ\nkiWsXr2aOXPmYK1l9OjRTJ8+nVatWrF69WreeustBg4cWF6vDFDIlspwYLv7h7ltqZtib+sSF7AD\nOW5/zUbQ+0oXrFsOUJ/eUBcdBx3OcpeRT8OmubDwXZj7upsNZthfodcVrr+3iIhIKV166aUADB48\nmH379rFnzx4AxowZQ3x8PPHx8QwdOpQ5c+Ywc+ZMJk+eTK9evQA4cOAAq1evplWrVrRu3brcAzb4\nFLKNMb8FHgI6A/2ttXOD9t0HXAvkArdZa7/xo0Yphdxs2LnKhehtXqjeugQObj96TK1m0KQbdDgb\nGnd1n2Lrt1fgClcREdCyv7v0uxa+vBe+uN11JznnKWjR1+8KRUSkOMrQ4lwWUVFRBAKBI/czMzOP\n3DYFvu3Ou1/Ydmst9913HzfccEO+fWlpadSsWbO8ywb8a8leApwPvBy80RjTBbgE6Ao0A74zxnSw\n1uZWfolSpIM7f906vWOF60sNEBkLjTq5vriNu7lg3bibugpUZ026w++/dH22v30AXhsGJ14OZzwI\nCY38rk5ERKqgxo0bs337dnbt2kVCQgKTJk1i+PDhAEyYMIGhQ4cyc+ZMEhMTSUxMBODzzz/nvvvu\n4+DBg0ydOpXHH3+c+Ph4HnjgAS677DISEhLYtGkT0dHRFVq7LyHbWrscfv1JAxgDvG+tzQLWGmNS\ngP7ArMqtUI7IzXHLkxdsnT6w9egxCU1ciG4/7Gigrt8eIiv2l1dCkDGu33bH4TD9SZj1Iiyf6Ppq\n97sOItWDTUREjoqOjuavf/0r/fv3p3nz5nTq1OnIvri4OHr16kV2djZvvPHGke09evRg6NCh7Ny5\nkwceeIBmzZrRrFkzli9fzkknnQRAQkIC77zzDpGRFfdNurHWVtjJj/vkxkwF7s7rLmKMGQfMtta+\n491/HfjKWvtRIY+9HrgeoFWrVn3WrVtXaXWHrYz0o63S25a4luodKyE3y+2PiIaGnY62Sudda9ly\nKa2dq+GreyE1GRp1gRH/hDaD/K5KRESA5cuX07lzZ7/LKNRpp53GU089Rd+++bsdPvTQQyQkJHD3\n3XeXy/MU9h4YY+ZZa4/b37HCmo2MMd8Bha29fL+19vOynt9a+wrwCkDfvn39+6QQinJzID3V6+6x\n5Gjr9P7NR4+p2ciF6LZDoHF3d7tBB7VOS/lqcAJc/olb4Oab++CtUW4A7FmPQGJzv6sTEREptQoL\n2dbaM0rxsE1A8NrYLbxtUlqHdh8N0dsWH+07neMNHIiIggYdXeth425HByOqj6xUFmOg8yjX3eiH\nZ2Hm07Dqaxh8N5x0ixa0ERGRX5k6dWqh2x966KFKraMoVa0D5ETgXWPMv3EDH08A5vhbUghbPgkm\nXA54Df01GrgW6X5/ONrdo0FHiIrxtUwRAKLj4bQ/Q89L4Zu/wJS/wYJ3YPgTbjpAERGREOLXFH7n\nAc8DDYH/GWMWWmvPttYuNcZ8ACwDcoCbNbNIGSz9FGrUh/NedoE6obEWd5Gqr25ruGQ8pEyBr/4E\n7/4WOoyA4f+Aem39rk5ERKRYfFn1w1r7qbW2hbU21lrb2Fp7dtC+R6217ay1Ha21X/lRX1gIBGDN\n99D+DDjhDKjVRAFbQkv7YXDjj3Dm3yBtBrwwEJIfgcMZflcmIiJyXFpaL1xt/QUydrmgIhKqomLg\nlNvhlrnQZYyb9u+F/rD0M/BxZiQREZHjUcgOVylT3HXbof7WIVIeajeFC16Fq7+EuET48Cr47xjY\nvsLvykREpAI999xzdO7cmcsuu+yYxyQkJABu9cZu3bpVVmnHpZAdrlKToUkPSGjodyUi5SfpFLh+\nmluSfctCeOkU+OZ+yNznd2UiIlIBXnzxRb799lvGjx/vdyklppAdjrL2w4afoN3pflciUv4io6D/\ndXDrfDjxMpj1AozrCwvfc2MRREQkLIwdO5Y1a9YwYsQIEhMTeeqpp47s69atG2lpaf4VVwxVbQo/\nKQ9rZ0AgR/2xJbzVbACjn4M+V8GX98BnY2Hef+CcJ6FpT7+rExEJK6u27Wd/Zna5nrNWXDQdGtc6\n5v6XXnqJr7/+mu+//55x48aV63NXBrVkh6PUZIiuCS0H+F2JSMVr3geu/Q7GvAC7UuHlITDpTshI\n97syERGpxtSSHY5Sp0DSqVopT6qPiAjodTl0GgVTH4M5r7p54of9FXpfBRGRflcoIhLSimpxrgxR\nUVEEgroEZmZm+lhN8aglO9ykr4X0NeoqItVTfB0Y8QSMnQGNuroW7VeHwvqf/K5MRETKICkpifnz\n5wMwf/581q5d63NFx6eQHW5Sk921Bj1Kdda4K1w9CS58Aw7sgDfOgk/Hwv5tflcmIiKlcMEFF5Ce\nnk7Xrl0ZN24cHTp08Luk41J3kXCTmgyJraB+e78rEfGXMdDtAjjhbJjxFPw4DpZPgqH3Qf/rITLa\n7wpFROQ4gmcQmTx5cqHHHDhwAHCt3UuWLKmMsopFLdnhJDcb1k6H9qdrCXWRPLEJcMZDcNNsaDUA\nvvkLvHQqrJnqc2EiIhLOFLLDyca5kLVPXUVECtOgPVz2EVzyHmQfcitGfnAl7Nngd2UiIhKGFLLD\nSeoUMBHQZojflYhUTcZAp3Pg5jkw9H5YNRnG9YPpT0J21R+pLiIioUMhO5ykJkPzvm6GBRE5tug4\nGHIv3DIHTjgTkh+BF/rDgvGQm+N3dSIiVYa11u8SfFPW166QHS4y0mHTfE3dJ1ISdVrBxW/DFZ9B\nXCJ8fhO80M8t0a6wLSLVXFxcHLt27aqWQdtay65du4iLiyv1OTS7SLhYMxWw0E4hW6TE2g2FttNh\n5Zfw/WNuifYZT8GQP7kZSrSYjYhUQy1atGDjxo3s2LHD71J8ERcXR4sWLUr9eIXscJE6xbXENevl\ndyUiockY6DQSOoyAFZNg6uPwyXWuv/aQP0HX8xS2RaRaiY6Opk2bNn6XEbLUXSQcWAspydD2NIjU\n5yaRMomIgC6jYexM+O1bYCLh42vh/50MSz6BoGV9RUREjkUhOxzsWAn7N2vqPpHyFBEBXX8DN/4I\nF/7Hbfvo9/DSKbD0M4VtEREpkkJ2OEid4q7VH1uk/EVEQLfzXdi+4HW36NOHV8HLg2D5F+6bJBER\nkQIUssNBajI06AB1WvpdiUj4ioiE7hfCzT/B+a+6BW0mXO7C9or/KWyLiEg+CtmhLjsT0n5QVxGR\nyhIRCT0ucgva/OYlOHwQ3v8dvHIarPxaYVtERACF7NC3/kfIOaSuIiKVLTIKTrwUbv4ZxrwImXvg\nvYvh1dPdSpIK2yIi1ZpCdqhLTYbIGEg6xe9KRKqnyCjodRncMhdGj4OMnfDub+G1M2D1dwrbIiLV\nlEJ2qEtJhlYDIaam35WIVG+R0dD7CrhlHpz7LBzYBuMvgNfPch+GFbZFRKoVhexQtn8rbF+qriIi\nVUlUDPS5Gm6dD6Oehn2b4O3z4I3hbmVWhW0RkWpBITuUpSa7aw16FKl6omKg7zVw2wI45ynYsx7+\nOwbeHAlrZ/hdnYiIVDCF7FCWMgVqNoLG3fyuRESOJSoW+l/nwvaIJyF9Dbw1Ct4c5WYGEhGRsKSQ\nHaoCAVjzPbQb6hbLEJGqLToOBlwPty2E4U/AzlXw5jnw1mhYP9vv6kREpJwpnYWqrb9Axi71xxYJ\nNdFxMHAs3P4LnP0P2L4M3jgb/vsb2DDH7+pERKScKGSHqiP9sYf6W4eIlE50PJx0M9y+CM56BLYu\nhtfPhHcugI3z/K5ORETKSCE7VKUkQ5PukNDI70pEpCxiasDJt8Idi+CMh2HzAnjtdBj/W9g03+/q\nRESklBSyQ1HWftgwW11FRMJJTE049Q7Xsj3sQdj4M7w6FN69GDYv9Ls6EREpIYXsUJQ2EwI5mrpP\nJBzFJsCgP7qwffr/uUGRrwyB934H25b6XZ2IiBSTQnYoSpkC0TXcSo8iEp7iasPge1w3kqH3w7qZ\n8NKp8PktsG+L39WJiMhxKGSHotRkSBrk5t8VkfAWlwhD7nWzkQy8CRZNgOd7w/ePQdYBv6sTEZFj\nUMgONbvTID1VXUVEqpv4unD2o3DzHOgwHKY9Ds/3gfn/hUCu39WJiEgBCtmhJmWKu26vQY8i1VK9\nNvDb/8C130Hd1jDxVteNJOU7vysTEZEgCtmhJjUZEltC/fZ+VyIifmrZD675Bi76L2QfcvNrv30e\nbF3id2UiIoJCdmjJzYa1011XEWP8rkZE/GYMdBnjupAMf9zNsf3SqfD5zRocKSLiM4XsULJpHmTt\nU1cREckvKgYG3gi3LXCrSC76wBsc+Q8NjhQR8YlCdihJmQImAtoM9rsSEamKfjU48gkXtue9qcGR\nIiKVTCE7lKROgeZ93X+kIiLHkm9wZBv44nbXjWT1d2Ct39WJiFQLCtmhIiMdNs3X1H0iUnwt+8E1\nX8NFb0NOJozPGxy52O/KRETCnkJ2qFgzFbDqjy0iJWMMdBkNN/3kBkduWQgvDYLPboZ9m/2uTkQk\nbClkh4rUZLfyW7PeflciIqEoeHDkybfA4g/cYjYaHCkiUiEUskOBtS5ktxkCkVF+VyMioSy+Lpz1\nCNzyM3Qc4QZHPtfLDY7MzfG7OhGRsKGQHQp2rIR9m9RVRETKT90kuPAN+MMUqNc2aHDktxocKSJS\nDhSyQ0FqsrvWoEcRKW8t+h4dHJmbBeMvhLd/o8GRIiJlpJAdClKnQP0ToE4rvysRkXCUb3DkE7Dl\nFw2OFBEpI4Xsqi47E9J+UFcREal4UTEwcCzcthBOvtUNjnyuNyQ/Cln7/a5ORCSkKGRXdetnQc4h\ndRURkcoTXwfO+rsbHNnpHJj+Txe25/5HgyNFRIpJIbuqS50CEdGQdKrflYhIdRM8OLJ+O5h0hwZH\niogUk0JjCByQAAAgAElEQVR2VZf6PbQaCDE1/a5ERKqrFn3h91/Bxe/kHxy5ZZHflYmIVFkK2VXZ\n/q2wbYn6Y4uI/4yBzufmHxz58mD47CYNjhQRKYRCdlWW+r27bqeQLSJVxK8GR37oDY58RIMjRUSC\nKGRXZalToGZDaNzN70pERPI7MjhyLnQaCdOfdCtH/vg8HD7od3UiIr5TyK6qAgG3CE270yFCPyYR\nqaLqtoYLX4c/JEOjLjD5/+CZ7jDzabVsi0i1pvRWVW1dBBm7NHWfiISGFn3gqolwzWRo1gu+e8iF\n7elPQuZev6sTEal0CtlVVeoUd62QLSKhpNUAuPxj17LdcoDrq/1Md5j6OBza7Xd1IiKVRiG7qkr9\nHpp0h4RGflciIlJyLfrA7ybA9dMgaRBMfQye6QFT/g4Z6X5XJyJS4RSyq6KsA7B+tlqxRST0NTsR\nLhkPY39wf9Nm/Mu1bH/7IBzc6Xd1IiIVxpeQbYx50hizwhizyBjzqTGmTtC++4wxKcaYlcaYs/2o\nz3dpMyCQran7RCR8NOkGF70FN82CDsPhh2dd2P7mfti/ze/qRETKnV8t2d8C3ay1PYBVwH0Axpgu\nwCVAV2A48KIxJtKnGv2TmgzRNdxKjyIi4aRRZzcbyc1zoPNomP0iPNsDvvqTFrURkbDiS8i21k62\n1uZ4d2cDLbzbY4D3rbVZ1tq1QArQ348afZUyBZJOhahYvysREakYDTvA+S+7eba7Xwg/vwbP9oT/\n3QV7NvhdnYhImVWFPtnXAF95t5sDwX9dN3rbfsUYc70xZq4xZu6OHTsquMRKtDsN0lPVVUREqof6\n7WDMC3DrPDjxdzDvLbeozRe3w+51flcnIlJqFRayjTHfGWOWFHIZE3TM/UAOML6k57fWvmKt7Wut\n7duwYcPyLN1fqcnuWoMeRaQ6qZsE5z4Lty2APlfBwnfh+d7w+c2wK9Xv6kRESiyqok5srT2jqP3G\nmKuBUcAwa631Nm8CWgYd1sLbVn2kTIHEltDgBL8rERGpfHVawsh/waC73ODIeW+6wN39Ihh8t/42\nikjI8Gt2keHAvcBoa21G0K6JwCXGmFhjTBvgBGCOHzX6IjcH1k6HdkPBGL+rERHxT+1mMOIJuH0R\nDLwJlk+Ecf3go2th+3K/qxMROS6/+mSPA2oB3xpjFhpjXgKw1i4FPgCWAV8DN1trc32qsfJtmgtZ\n+9QfW0QkT63GcPajLmyfcjus/ApePAk+uAq2LvG7OhGRY6qw7iJFsda2L2Lfo8CjlVhO1ZGaDCYC\n2g7xuxIRkaoloSGc+TCcfJub9u+nl2HZZ9BpFAy+xy16IyJShVSF2UUkT8oUaN4H4uv6XYmISNVU\nsz4MewDuXAyn3ecW73plCLx7MWyc53d1IiJHKGRXFRnpsHm+uoqIiBRHfF047c9wx2I4/f9gw0/w\n2unwzgWw/ie/qxMRUciuMtZOAxvQ1H0iIiURl+i6i9yxGM54CDYvgDfOgrdGQ9oPflcnItWYQnZV\nkTIFYhNddxERESmZ2Fpw6p0ubJ/1iJuB5M1z4D8jYc00ODJTrIhI5VDIrgqshdTv3YDHSF/GooqI\nhIeYmnDyrXDHIhj+hFtB97+j4Y3hrjFDYVtEKokSXVWwcxXs2+gWWhARkbKLjoeBY6HP1bDgbZj5\nNLxzvuvLHVcH4mpDbG3X3SS2dtD9gtcF9kfHax0DESkWheyqIGWKu1Z/bBGR8hUdB/2vg95XwqIJ\nsHmhW48gc5+7Tl+b/z7HaemOiCo6hBd6nZj/fnQNBXWRakAhuypITYb67aFua78rEREJT1GxLmj3\nvvLYxwQCcPjA0dCduTcogO89GsQLXu9OK0VQr3XsEB4RDRER7jgTCRGR3u2IoNt52yMLHFPU9nI6\nZ3S8uy0iRVLI9lt2JqTNLPoPv4iIVLyICBdy42pDYinPUTCo57vee4zt+2DP+qP3AzkQyHXXNtfN\nPFWVxNaGDmdD53Oh/RmuH7yI/IpCtt82zIacQ9Be82OLiIS88gjqBVnrQrfNzR++A4Gg23nbAwWO\nyT3GY4+1vRjn3LkKVnwJiz+EqDgXtDuf64K3FlMTOUIh228pU9xXg61P8bsSERGpiozxZp6qQv9l\nj8qB9T/C8i9g+SRYMcl1LWkz2AXujiOhVmO/qxTxlbFhMJ1R37597dy5c/0uo3T+3ynuk//Vk/yu\nREREpOQCAbdi8fKJsGwi7F4LGGg10AXuTqM05kjCijFmnrW27/GOq0Ifi6uh/Vth2xIY9qDflYiI\niJRORAS06OsuZzwM25d5LdxfwDd/cZemPV3g7jwaGnb0u2KRSqGQ7afU7921+mOLiEg4MAYad3WX\n0/4Mu1JdV5LlX0DyI+7SoMPRwN20p6YzlLClkO2n1GSo2RAad/e7EhERkfJXvx2ccru77NsMK/7n\nupXMfAZm/AsSW3mB+1xo2V9TA0pYUcj2SyDgQna7091XbSIiIuGsdjO3MFD/6+DgLlj1lWvh/vlV\nmP0C1GwEnUa6wJ00CKJi/K5YpEwUsv2ydRFk7FRXERERqX5q1odel7tL5j5I+dYNmlz0Acz7j1tJ\ns8MIF7jbnQ4xNfyuWKTEFLL9kprsrtsO9bcOERERP8XVhm4XuEv2ITdeafkXsPJLWPS+W4a+/Rmu\nD3eHs1wAFwkBCtl+SU12fbE1j6iIiIgTHQ+dznGX3Gy3IvLyL7zBkxPduhJtT/OmBhwJNRv4XbHI\nMWmebD9kHYAnkuCkm+DMv/ldjYiISNUWCMDGn13QXv4F7FkHJsIt5JYXuBNb+F2lVBOaJ7sqS5sJ\ngWzXz0xERESKFhEBrQa4y1mPwNbFR+fi/uped2nexwvc50KdloAJmh4w77Z3rWkDpRKERcjOOJzD\nvHXpfpdRfPN/BtMN6AyhVLeIVFvWukvAWgJYbMC7HbzdWqx3sAUCFvD2We8ckPcY71iOnsNavOPs\nkecMeLfznwMCAe8+1nsOd9vmO7drAM07Z97zlea1l0bof098DN57DUffm8Je69F9BY61wcf8+qdy\n5KdX4Nx5vx/uTjRwPjbpfGo23kyj9Lk0Tp9LncnjYfL4ErwUg8UL30eeKyiQ5x1nvO15x3v3g4+3\nBAf3oGM5eiobdO78xwc/nSm44dfHFtxuTJHHuddQ8LwFrgs8NPjcBc979D3L//z59hf2uHzva8Fa\nTeGPM8Gvy3vvvLtxA/9AnWbtC3m1VUOxQ7Yxpoa1NqMii6k2Ns1zE/BHRvtdiYhUkkAAsnJyycwJ\nkJUdICsnl6zsXLJyAmRmB8jMyfW2BcjKcfszs92+LG9fbi4E8kJpwIXdQMB6YRQv/Lr9eQEYC7lY\nrD16bK6XVnLzwnDA7c8NCsUuTHPkfGEbGCVMdAO60YA99IlYRQ0yj8RcCI601jVk593Ot98WiH/B\n+/nVbXPkX0Vw5Mx/TvI9hiPbg8+Vd45g+eO6LXCd/zG/3p7/cYVtM8e4f1T+9y34XPlf55GPswXe\nn7zPIQVrPNbzBz/Pr5/r6GPy15Ox9yB1mv2q+CrjuCHbGHMy8BqQALQyxvQEbrDW3lTRxRVXjZgo\n+rSu53cZxbN7HRz4Hk59HEKlZpFqwFpLVk6AQ4dzycjO5dDhHA4dDpBxOMe7n0vGYW97dt5td51x\nOJdD2TlHj8m33x2fmR0oUT3GQI3oSOJjooiPiaBGdBTRUYYI4y6RkYbIiAiMMUQaQ0QER/dFGCIM\nx9xnDG67MUR4x7rHBO2LyHsuvMd45zIc2RcZdN687cYYDBw5V4QBg/G+oc+ry+0n33HucXnHmQL7\nCDomb7vxzh1R4NyFPp6jx5emp0BpexeYIloWQ9nRoBr83phC9pmgPUePLezncGSfMYUeX/DcFLVP\nqoVacdHERFXdtUaK05L9NHA2MBHAWvuLMWZwhVYVzvKm7mun+bFFKktuwLJjfxab9x5iy55Mtuw9\nxOa8672ZbNlziF0HD5MbKFl7bWxUBDViIqkRE0VcdAQ1YqKIj4mkXs0YWtSNJD7aC8gxUcRHR3rH\nesE5Ou/20e1x0e5cNWIiiY2KOBJQREQk9BSru4i1dkOBP/a5FVNONZA6BWq3gAYn+F2JSFiw1rLr\n4GG27Mn0QvQhtuzNPBKet+zNZNu+THIKBOj46Eia1omjeZ14OnZsSMNasUcCbsHAeyQIR0cRFxSa\nIyMUgkVEpHDFCdkbvC4j1hgTDdwOLK/YssJUbg6smQ5dx2hks0gxWGvZdyjHheeg1ucjgXpvJlv2\nZnI4J39XjJjICJokxtE0MY7+berRNDGOpnXiaZYYR9PEeJrViSMxPlotxSIiUmGKE7LHAs8CzYFN\nwGSgyvTHDimb5kHWXk3dJ+I5mJWTv+tGXojem8lmrxU643D+L84iIwyNa8XStE483ZsncnbXJi5E\ne+G5aWI89WvGEKFWZhER8VFxQnZHa+1lwRuMMacAP1RMSWEsdQqYCLdalUg1sXF3BvPW7WZDeka+\nLhyb9xxiX2bOr45vWCuWZolxnNCoFoM7NKRZYjxN6xwN0Q0TYomKrLoDXURERKB4Ift5oHcxtsnx\npCa7yfLj6/pdiUiFCAQsKTsOMGdtOj+npfPz2nQ27808sr9OjWiaJsbTvE48/ZLq0bROnAvRiXE0\nqxNP49pxVXqkuIiISHEdM2QbY04CTgYaGmP+GLSrNhBZ0YWFnUO7XXeRwff4XYlIucnODbB4017m\npqUzZ+1u5q5LZ09GNuBapPsn1eP6pLr0TapHu4YJxMfoT4eIiFQPRbVkx+Dmxo4CagVt3wdcWJFF\nhaU1U8EGNHWfhLSMwzksWL+Hn9a6VuoFG3Yfmf85qX4NzuzcmH5t6tE/qR6t69fQwEIREam2jhmy\nrbXTgGnGmDettesqsabwlJoMsYmuu4hIiEg/ePhIt4+f1+1m6aa95ATcimmdm9Tmkn6t6JdUj35J\ndWlUO87vckVERKqM4vTJzjDGPAl0BY78L2qt1RQZxWUtpCRD28EQWeyV7EUq3cbdGfzsdf34OS2d\nlO0HADclXs+WiVw/uC392tSjT+u61I6L9rlaERGRqqs4iW88MAEYhZvO7ypgR0UWFXZ2roZ9G2Hw\n3X5XInJEUYMUa8VG0bt1Xc7r1Zx+SfXo0SKRuGj1pxYRESmu4oTs+tba140xtwd1Ifm5ogsLK6lT\n3LXmxxYfZecGWLJp75GW6nnr0tntDVJskBBL/zZ1uS6pHv2S6tG5aW2tZigiIlIGxQnZ2d71FmPM\nSGAzUK/iSgpDKVOgfnuo29rvSqQayRukmNdSvWD9Hg5lu4VdWtevwbDOjemfVI9+beqRpEGKIiIi\n5ao4IfsRY0wicBdufuzawJ0VWlU4ycmCtJnQ+wq/K5EwtztvkGJaOnPSfj1I8eJ+LTVIUUREpJIc\nN2Rbayd5N/cCQyu2nDC0fhbkHNLUfVIhcnIDvD5zLR/N28hqDVIUERGpMo4bso0xbYBbgaTg4621\noyuurDCSmgwR0ZB0qt+VSJhZuXU/93z0C4s27mVAm3rcc3ZHDVIUERGpIorTXeQz4HXgCyBQseWE\noZRkaDUQYhP8rkTCRHZugJempvJc8mpqx0Xzwu96M7JHU7/LEhERkSDFCdmZ1trnKryScLR/G2xb\nDMMe9LsSCRPLNu/jno9+YenmfZzbsxkPnduF+gmxfpclIiIiBRQnZD9rjHkQmAxk5W201s6vsKrC\nxZrv3bWm7pMyOpwT4MWpKYxLTqFOjRheurwPw7s18bssEREROYbihOzuwBXA6RztLmK9+1KUlClQ\nowE06eF3JRLClmzay90f/sKKrfv5zYnNePDcrtStGeN3WSIiIlKE4oTs3wJtrbWHK7qYsBIIuJbs\ndqdDRITf1UgIysrJZVxyCi9OTaV+zRhevbIvZ3Zp7HdZIiIiUgzFCdlLgDrA9gquJbxsWwwHd6ir\niJTKoo17uOfDRazctp8Lerfgr6O6kFhD0/CJiIiEiuKE7DrACm8p9eA+2ZrCrygpeUupa2pxKb7M\n7Fyem7Kal6evoWFCLG9c3ZfTO6n1WkREJNQUJ2RraozSSE2Gxt2glganSfEsWL+bez5aRMr2A1zU\ntwX3j+xCYrxar0VEREJRcVZ8nFYZhYSVrAOwfjYMvNHvSiQEZGbn8vS3q3h1xhoa147jrWv6M6RD\nQ7/LEhERkTI4Zsg2xsy01p5qjNmPm03kyC7AWmtrV3h1oWrdDxDIhvZaSl2KNm9dOvd8tIg1Ow5y\naf9W/OWcTtTSEugiIiIh75gh21p7qnddq/LKCRMpUyAqHloO9LsSqaIOHc7lqckreeOHtTRLjOed\nawdw6gkN/C5LREREyslxu4sYY9621l5xvG0SJHUKJJ0K0XF+VyJV0Jy16dz70S+k7crgioGt+dOI\nTiTEFmd4hIiIiISK4vzP3jX4jjEmCuhTMeWEgd3rYFcK9L3W70qkisk4nMM/v17JW7PSaFE3nnev\nG8DJ7dR6LSIiEo6K6pN9H/AXIN4Ysy9vM3AYeKUSagtNqcnuWv2xJcis1F386eNFrE/P4OqTk7jn\n7I7UVOu1iIhI2CqqT/ZjwGPGmMestfdVYk2hLTUZareABh38rkSqgINZOTz+1Qrenr2O1vVrMOH6\ngQxoW9/vskRERKSCFacpbZIxpqa19qAx5nKgN/CstXZdBdcWenJzYM006DIajPG7GvHZDyk7+dPH\ni9i05xDXnNKGe87uSHxMpN9liYiISCUoTsj+f0BPY0xP4C7gNeC/wJCKLCwkbZoHWXvVVaSa25+Z\nzWNfreDdn9bTtkFNPrzhJPom1fO7LBEREalExQnZOdZaa4wZA4yz1r5ujNGovsKkJoOJgDb6/FFd\nTV+1gz9/vIit+zK5fnBb/nhmB+Ki1XotIiJS3RQnZO/3BkFeAQwyxkQAWi2jMKlToFlvqKFWy+pm\nX2Y2j05azoS5G2jXsCYf3XgyvVvV9bssERER8UlxQvbFwO+Aa6y1W40xrYAnK7asEHRot+suMvge\nvyuRSvb9yu385ZPFbNuXyY2nteP2YSeo9VpERKSaO27I9oL1x8AJ3qadwKcVWlUoWjMNbADane53\nJVJJ9mZk87dJy/h4/kY6NE7gpctPoWfLOn6XJSIiIlVAcVZ8vA64HqgHtAOaAy8BGt0XLHUKxNaG\n5n39rkQqwXfLtvGXTxez6+BhbhnanluHtSc2Sq3XIiIi4hSnu8jNQH/gJwBr7WpjTKOyPKkx5u/A\nGCAAbAeuttZu9vbdB1wL5AK3WWu/KctzVQprIfV7aDMYIrXASDjbk3GYh79YxqcLNtGpSS1ev6of\n3Vsk+l2WiIiIVDHFSYRZ1trDxpv32VtW3ZbxeZ+01j7gne824K/AWGNMF+AS3FLuzYDvjDEdrLW5\nZXy+irVzNezdAIP+6HclUoG+WbqV+z9dwp6Mw9w+7ARuHtqemKgIv8sSERGRKqg4IXuaMSZvefUz\ngZuAL8rypNbafUF3a3I0tI8B3rfWZgFrjTEpuFb0WWV5vgqXt5S6+mOHpfSDh3lo4lIm/rKZLk1r\n89Y1/ejaTK3XIiIicmzFCdl/xnXfWAzcAHyJW5CmTIwxjwJXAnuBod7m5sDsoMM2etsKe/z1uL7i\ntGrVqqzllE3qFKjXDuom+VuHlLtV2/bzu1dns/dQNn88swM3ntaO6Ei1XouIiEjRijO7SAB41bsU\nmzHmO6BJIbvut9Z+bq29H7jf64N9C/BgSc5vrX0FeAWgb9++Ze2+Uno5WZA2E3pd7lsJUjEOZOUw\n9p15gOGLW0+lU5PafpckIiIiIaLCRulZa88o5qHjca3jDwKbgJZB+1p426qu9bMhO0NdRcKMtZb7\nPllM2s6DjP/DQAVsERERKRFfvvc2xpwQdHcMsMK7PRG4xBgTa4xpg5ube05l11ciqVMgIhqSBvld\niZSjd2av44tfNnPXWR05qV19v8sRERGREFOqlmxjzFPW2rvL8LyPG2M64qbwWweMBbDWLjXGfAAs\nA3KAm6v8zCKpydBqIMQm+F2JlJNFG/fw90nLGdqxITcOaed3OSIiIhKCSttd5CKg1CHbWntBEfse\nBR4t7bkr1YHtsHUxDPur35VIOdmbkc1N4+fTsFYs/77oRCIijN8liYiISAgqbchW8oCgqfu0+GU4\nCAQsd324kG37MvnghpOoWzPG75JEREQkRB0zZBtj6h1rFwrZTmoy1GgATXr4XYmUg1dmrOG75dt5\n8Nwu9GpV1+9yREREJIQV1ZI9D7dITGGBOrtiygkhgYAL2e2GQoTmTQ51P63ZxZPfrGRk96ZcfXKS\n3+WIiIhIiDtmyLbWtqnMQkJOdgZ0Hg3t1VUk1O3Yn8Wt7y2gVb0aPH5Bd4zRFzUiIiJSNkV1F+ld\n1AOttfPLv5wQEpsAo/7tdxVSRrkBy+3vL2DvoWzeuqY/teKi/S5JREREwkBR3UXmAkuAnd794OY9\nC2j1FQl5z363ih9Td/HPC3rQuakWnBEREZHyUVTI/iNwIXAIeB/41Fp7oFKqEqkE01bt4PnvU7ig\ndwt+27eF3+WIiIhIGDnmiD1r7TPW2lOBW3FLnU8xxnxgjDmx0qoTqSCb9xzijvcX0KFRLR75TTf1\nwxYREZFyddxpMay1a4DPgclAf6BDRRclUpGycwPc8u58DucEePHy3sTHRPpdkoiIiISZogY+tgUu\nAcYAG3BdRv5hrT1USbWJVIjHv1rB/PV7eP7SXrRrmOB3OSIiIhKGiuqTnQIswrVi7wNaATfmfa1u\nrdXUGhJyvl6yhddnruXKk1pzbs9mfpcjIiIiYaqokP033CwiAAWb+ywiISZt50Hu+XARPVskcv/I\nzn6XIyIiImGsqMVoHjrWPmPMHRVSjUgFyczO5abx84mIMIz7XW9io9QPW0RERCpOadcD/2O5ViFS\nwR7+YinLtuzj3xf1pGW9Gn6XIyIiImGutCFb851JyPhk/kbem7OBsUPaMaxzY7/LERERkWqgtCFb\nfbIlJKzatp/7P11C/zb1uPsszT4pIiIilaOoKfz2U3iYNkB8hVUkUk4OZuVw0/j51IyNZNylvYiK\nLO1nShEREZGSKWrgY63KLESkPFlr+cuni1mz4wDvXDuARrXj/C5JREREqhE17UlYGv/Tej5fuJk7\nz+jAye0b+F2OiIiIVDMK2RJ2Fm/cy9++WMaQDg25eWh7v8sRERGRakghW8LK3oxsbnp3HvUTYnj6\n4hOJiNBEOCIiIlL5ilrxUSSkWGu5+6Nf2LInkwk3nES9mjF+lyQiIiLVlFqyJWy8OmMN3y7bxn3n\ndKZP67p+lyMiIiLVmEK2hIWf09J54uuVjOjWhGtOSfK7HBEREanmFLIl5O08kMUt786nZd14nriw\nB8aoH7aIiIj4SyFbQlpuwHLH+wvZnZHNC5f1pnZctN8liYiIiChkS2h7bspqZqbs5G+ju9K1WaLf\n5YiIiIgACtkSwqav2sFzyas5v3dzLu7X0u9yRERERI5QyJaQtGXvIe6YsJATGiXwyG+6qR+2iIiI\nVCkK2RJysnMD3PLuArKyc3nxsj7UiNF07yIiIlK1KJ1IyPnn1yuYt243z13ai/aNEvwuR0RERORX\n1JItIeWbpVt5dcZarhjYmtE9m/ldjoiIiEihFLIlZKzflcHdH/5CjxaJ/N+ozn6XIyIiInJMCtkS\nEjKzc7lx/DwM8MLvehMbFel3SSIiIiLHpD7ZEhL+NmkZSzfv47Ur+9KyXg2/yxEREREpklqypcr7\nbMEm3v1pPTcMacsZXRr7XY6IiIjIcSlkS5W2ett+7vtkMf2S6nL3WR39LkdERESkWBSypco6mJXD\njePnUyMmkucv7U10pH5dRUREJDSoT7ZUSdZa7v90Mak7DvD2NQNokhjnd0kiIiIixaamQamS3p2z\nns8WbuaOYR049YQGfpcjIiIiUiIK2VLlLNm0l4cnLmPQCQ249fT2fpcjIiIiUmIK2VKl7D2UzY3j\n51GvZgzPXHwiERHG75JERERESkx9sqXKsNZyz4e/sGVPJhNuGEj9hFi/SxIREREpFbVkS5Xx2oy1\nTF62jT+P6ESf1vX8LkdERESk1BSypUqYm5bO41+v4Oyujbn21DZ+lyMiIiJSJgrZ4rtdB7K45d0F\nNK8Tzz8v7Ikx6octIiIioU19ssVXuQHLHRMWkp5xmE9uPJnE+Gi/SxIREREpM4Vs8c3BrByemryS\nGat38o/zutOteaLfJYmIiIiUC4VsqXQ79mfx5o9reXvWOvZl5nBp/1Zc2r+l32WJiIiIlBuFbKk0\na3Yc4NUZa/l4/kaycwMM79qE6we3pVerun6XJiIiIlKuFLKlwi1Yv5uXp63hm2VbiY6M4MI+Lbhu\nUFvaNKjpd2kiIiIiFUIhWypEIGCZumo7L01bw5y16dSOi+Lm09pz1clJNKylRWZEREQkvClkS7k6\nnBNg4i+beWV6Kqu2HaBZYhwPjOrCxf1akhCrXzcRERGpHpR6pFzsz8zm/TkbeH3mWrbuy6RTk1o8\nfXFPRvVoRnSkpmMXERGR6kUhW8pk+75M/vNjGu/MXsf+zBxOalufxy/ozpAODbWojIiIiFRbCtlS\nKinbD/Dq9DV8umATOYEAI7o35YbBbenRoo7fpYmIiIj4TiFbSmTeunRemraGb5dtIzYqgov7teQP\ng9rQur5mChERERHJo5AtxxUIWKas2M7L01KZu243dWpEc9v/b+/eo6wq7zyNPz9uchERxKhc5KKA\nEVEiYDBtjElYauKFtNMmdjuMOi6NWZlcXGntoB2j7WiMxmXSq6ftGFtJJmltx8FLa+KNJF4ShRgv\ngKhBBBEEwRBBUAqr6jd/nO2kGquEqjpVu07xfNZi1Tn77LP397wU1Lf22efdnx7HGUeOYq/dnSlE\nkiRpe5ZstaiuvoG7nn6NHz6yjGXrtzB8z35cetLBfH7aSPr38VtHkiSpJTYlvc+mre/yb/NXctNj\ny6FjEdwAABM3SURBVFn3Vh0H77cHPzhtMidM2o9ezhQiSZK0Q5Zs/X9rN27l5t8s52fzV7K5rp6j\nDhzKtZ8/jKMOHOpMIZIkSa1gyRZLX3+LGx55mTufWU1DY3LCocP44tFjOWT4oLKjSZIk1aRSS3ZE\nfAP4HrB3Zr5RLJsNnA00AF/NzPtLjNhtZSa/W/EnfvjwMua9sI6+vXtw+kdHcfZRYxg5pH/Z8SRJ\nkmpaaSU7IkYCxwIrmyw7GDgNmAgMAx6KiPGZ2VBOyu6nsTF5YMnr/PCRZTy98k2GDOjD+TPGM+vI\nUQwZ0KfseJIkSd1CmUeyrwMuBO5qsmwmcGtm1gHLI+Il4Ajg8RLydStb323gjqdX86NHXublN7Yw\nckg/Lp85kb+aMpJ+fXqWHU+SJKlbKaVkR8RMYHVmPrvdB+qGA080ub+qWNbcNs4FzgXYf//9Oyhp\n7dv49rv8dP4r3PybFbyxuY5JwwfxT3/zEY6fuK8zhUiSJHWQDivZEfEQsG8zD10MXETlVJE2y8wb\ngBsApk6dmu3ZVnf16NL1nPe/f8+WbQ0cPX5vzjt6LEcesJczhUiSJHWwDivZmTmjueURMQkYA7x3\nFHsE8FREHAGsBkY2WX1EsUyttLmungtvX8h+e/bjH0/7CAcP26PsSJIkSbuMTj9fIDMXZeaHMnN0\nZo6mckrI4Zm5FrgbOC0idouIMcA4YEFnZ+wOvnf/i6zdtJXv/pdDLdiSJEmdrEvNk52Zz0XEbcAS\noB74sjOLtN4zr77Jjx9fwazpo5gyanDZcSRJknY5pZfs4mh20/tXAFeUk6b2vdvQyDf/70L2GdiX\nC46bUHYcSZKkXVLpJVvVdeOjy3lh7VvcMGsKA/v2LjuOJEnSLsk53LqRV/64he8/9AeOn7gvx05s\nbmIXSZIkdQZLdjeRmVx8x2L69OzBpSdPLDuOJEnSLs2S3U3c8fRqHnvpDS78zEHsO6hv2XEkSZJ2\naZbsbuCPm+u4/J4lTBk1mNOP8OqXkiRJZbNkdwNX3Ps8m+vq+c4pk+jRw6s5SpIklc2SXeMeXbqe\nuU+v5rxPHMD4fQaWHUeSJElYsmvaO9sauPiOxYwdOoAvf/LAsuNIkiSp4DzZNewH85aycsPb3Hru\ndPr27ll2HEmSJBU8kl2jlry2iR89+jJfmDqS6WP3KjuOJEmSmrBk16CGxmT23IUM7t+b2Z89qOw4\nkiRJ2o4luwb95PEVPLtqI5ecNJE9+/cpO44kSZK2Y8muMavffIdr7n+RYybszUmH7ld2HEmSJDXD\nkl1DMpNL7lxMJlw+8xAinBNbkiSpK7Jk15CfL1rLvBfW8Y1jxzNySP+y40iSJKkFluwasfHtd/n2\n3c8xafggzvzY6LLjSJIk6QM4T3aNuOq+F/jT29uYc9Y0evX0dyNJkqSuzLZWAxYs38AtC1Zy9lFj\nOGT4oLLjSJIkaQcs2V1cXX0Ds+cuZMTgfnx9xriy40iSJGkneLpIF3f9r5exbP0W5pw1jf59/OuS\nJEmqBR7J7sJeWvcW//yrZcycPIxjJnyo7DiSJEnaSZbsLqqxMblo7mL69enJt048uOw4kiRJagVL\ndhd16+9eZcGKDVx8wocZuvtuZceRJElSK1iyu6B1m7bynV88z5Fj9+LUKSPKjiNJkqRWsmR3QZf9\nxxLq6hu58pRJXjpdkiSpBlmyu5iHlrzOvYvW8LVPj2PM0AFlx5EkSVIbWLK7kM119Vxy12Im7DOQ\ncz4+tuw4kiRJaiNLdhdy7QMvsmbTVq48ZRJ9evlXI0mSVKtscl3EM6++yZzfrmDW9FFMGTW47DiS\nJElqB0t2F/BuQyOz5y5in4F9ueC4CWXHkSRJUjt5ne4u4MZHl/P8mk38cNYUBvbtXXYcSZIktZNH\nskv2yh+38P2H/sDxE/fluIn7lh1HkiRJVWDJLlFmcvEdi+nTsweXnjyx7DiSJEmqEkt2ie54ejWP\nvfQGF37mIPYd1LfsOJIkSaoSS3ZJNmzZxuX3LGHKqMGcfsT+ZceRJElSFVmyS/I/713C5rp6vnPK\nJHr08NLpkiRJ3YkluwSPLX2DuU+t5rxPHMD4fQaWHUeSJElVZsnuZO9sa+CiOxYxZugAvvzJA8uO\nI0mSpA7gPNmd7B9/uZSVG97mlnOm07d3z7LjSJIkqQN4JLsTLXltEzc88jJfmDqSIw/Yq+w4kiRJ\n6iCW7E7S0JjMnruQwf17M/uzB5UdR5IkSR3Ikt1JfvL4Cp5dtZFLTprInv37lB1HkiRJHciS3QlW\nv/kO19z/IsdM2JuTDt2v7DiSJEnqYJbsDpaZXHLnYjLh8pmHEOGc2JIkSd2dJbuD/WLxWua9sI5v\nHDuekUP6lx1HkiRJncCS3YE2vvMu3777OQ4Zvgdnfmx02XEkSZLUSZwnuwN9974X2LBlGzefOY1e\nPf19RpIkaVdh8+sgC5Zv4N/mr+Tso8ZwyPBBZceRJElSJ7Jkd4C6+gZmz13IiMH9+PqMcWXHkSRJ\nUifzdJEOcP2vl7Fs/RbmnDWN/n0cYkmSpF2NR7Kr7KV1b/HPv1rGzMnDOGbCh8qOI0mSpBJYsquo\nsTG5aO5i+vXpybdOPLjsOJIkSSqJJbuK/v3JV1mwYgMXn/Bhhu6+W9lxJEmSVBJLdpWs27SVK3/+\nPNPHDuHUKSPKjiNJkqQSWbKr5LJ7llBX38iVfznJS6dLkiTt4izZVTDv+de5d+EavvqpAxm79+5l\nx5EkSVLJLNnttLmunm/duZgJ+wzk3KMPKDuOJEmSugBLdjtd+8CLrNm0lStPmUSfXg6nJEmSLNnt\n8syrbzLntyuYNX0UU0YNLjuOJEmSughLdhu929DI7LmL2GdgXy44bkLZcSRJktSFeM3vNtpSV8+w\nQX35+oxxDOzbu+w4kiRJ6kIs2W20Z/8+3HjGVKfrkyRJ0vuUcrpIRFwaEasj4pniz2ebPDY7Il6K\niBcj4rgy8u0sC7YkSZKaU+aR7Osy83tNF0TEwcBpwERgGPBQRIzPzIYyAkqSJElt0dU++DgTuDUz\n6zJzOfAScETJmSRJkqRWKbNkfyUiFkbETRHx3vx3w4FXm6yzqlgmSZIk1YwOK9kR8VBELG7mz0zg\nemAsMBlYA1zbhu2fGxFPRsST69evr3J6SZIkqe067JzszJyxM+tFxI+Ae4q7q4GRTR4eUSxrbvs3\nADcATJ06NdueVJIkSaqusmYX2a/J3b8EFhe37wZOi4jdImIMMA5Y0Nn5JEmSpPYoa3aRqyNiMpDA\nCuCLAJn5XETcBiwB6oEvO7OIJEmSak0pJTszZ33AY1cAV3RiHEmSJKmqutoUfpIkSVLNs2RLkiRJ\nVWbJliRJkqrMki1JkiRVWWTW/hTTEbEeeKWk3Q8F3ihp392FY9h+jmH7OYbt5xhWh+PYfo5h+zmG\nLRuVmXvvaKVuUbLLFBFPZubUsnPUMsew/RzD9nMM288xrA7Hsf0cw/ZzDNvP00UkSZKkKrNkS5Ik\nSVVmyW6/G8oO0A04hu3nGLafY9h+jmF1OI7t5xi2n2PYTp6TLUmSJFWZR7IlSZKkKrNkt0NE9IyI\npyPinrKz1KKI2DMibo+IFyLi+Yg4suxMtSgizo+I5yJicUTcEhF9y87U1UXETRGxLiIWN1k2JCIe\njIilxdfBZWbs6loYw2uKf88LI+KOiNizzIxdXXNj2OSxb0RERsTQMrLVipbGMCK+UnwvPhcRV5eV\nr1a08O95ckQ8ERHPRMSTEXFEmRlrkSW7fb4GPF92iBr2A+C+zDwIOAzHstUiYjjwVWBqZh4C9ARO\nKzdVTZgDHL/dsm8C8zJzHDCvuK+WzeH9Y/ggcEhmHgr8AZjd2aFqzBzeP4ZExEjgWGBlZweqQXPY\nbgwj4pPATOCwzJwIfK+EXLVmDu//XrwauCwzJwOXFPfVCpbsNoqIEcAJwI1lZ6lFETEIOBr4V4DM\n3JaZb5abqmb1AvpFRC+gP/BayXm6vMx8BNiw3eKZwI+L2z8GPtepoWpMc2OYmQ9kZn1x9wlgRKcH\nqyEtfB8CXAdcCPihqR1oYQy/BFyVmXXFOus6PViNaWEcE9ijuD0If7a0miW77b5P5T/BxrKD1Kgx\nwHrg5uKUmxsjYkDZoWpNZq6mcpRmJbAG2JiZD5Sbqmbtk5lrittrgX3KDNMN/HfgF2WHqDURMRNY\nnZnPlp2lho0HPh4R8yPi4YiYVnagGvV14JqIeJXKzxnfmWolS3YbRMSJwLrM/H3ZWWpYL+Bw4PrM\n/AiwBd+eb7XivOGZVH5pGQYMiIj/Wm6q2peVaZc8ithGEXExUA/8rOwstSQi+gMXUXlrXm3XCxgC\nTAcuAG6LiCg3Uk36EnB+Zo4Ezqd451k7z5LdNn8BnBwRK4BbgU9FxE/LjVRzVgGrMnN+cf92KqVb\nrTMDWJ6Z6zPzXWAu8LGSM9Wq1yNiP4Diq28xt0FEnAmcCJyezhHbWgdQ+YX52eLnywjgqYjYt9RU\ntWcVMDcrFlB5x9kPkLbeGVR+pgD8H8APPraSJbsNMnN2Zo7IzNFUPmT2y8z06GErZOZa4NWImFAs\n+jSwpMRItWolMD0i+hdHaj6NHyBtq7up/FCh+HpXiVlqUkQcT+U0upMz8+2y89SazFyUmR/KzNHF\nz5dVwOHF/5faeXcCnwSIiPFAH+CNUhPVpteATxS3PwUsLTFLTepVdgDt0r4C/Cwi+gAvA2eVnKfm\nZOb8iLgdeIrK2/NP41W6digibgGOAYZGxCrg28BVVN5WPht4Bfh8eQm7vhbGcDawG/Bg8e78E5l5\nXmkhu7jmxjAzfUu+FVr4PrwJuKmYjm4bcIbvqnywFsbxHOAHxYfqtwLnlpewNnnFR0mSJKnKPF1E\nkiRJqjJLtiRJklRllmxJkiSpyizZkiRJUpVZsiVJkqQqs2RL2iVFREbEtU3u/21EXFqlbc+JiL+q\nxrZ2sJ9TI+L5iPjVdsuHFVM7tmZbZ0bEP7Uxxz9ExIy2PLe9dmbfEXFMRHiRJkmdynmyJe2q6oBT\nIuI7mdllLlQREb0ys34nVz8bOCczH2u6MDNfAzq85DfZX2mXAd/JfR8DbAZ+27FpJOnPPJItaVdV\nT+XCPedv/8D2R6IjYnPx9ZiIeDgi7oqIlyPiqog4PSIWRMSiiDigyWZmRMSTEfGHiDixeH7PiLgm\nIn4XEQsj4otNtvtoRNxNM1c+jYi/Lra/OCK+Wyy7BDgK+NeIuGa79UcXF+J47wj13Ii4LyKWRsTV\nTdY7q8i3APiLHb3+4vbfFVmejYirtl8/IlZExGUR8VSx3kHF8r0j4sGIeC4iboyIVyLifZe6jojN\nEXFdsd68iNi7WD45Ip4oxu2OiBi8M/uOiNHAecD5EfFMRHy8eAdgcfEaHtk+gyRVgyVb0q7sfwGn\nR8SgVjznMCql7cPALGB8Zh4B3EjlKqbvGQ0cAZwA/EtE9KVy5HljZk4DpgHnRMSYYv3Dga9l5vim\nO4uIYcB3qVzWeDIwLSI+l5n/ADwJnJ6ZF+wg82TgC8Ak4AsRMTIi9gMuo1KujwIO3tELj4jPADOB\nj2bmYcDVLaz6RmYeDlwP/G2x7NvALzNzInA7sH8Lzx0APFms93DxPICfAH+XmYcCi5os/8B9Z+YK\n4F+A6zJzcmY+ClwCHFe8hpN39LolqS0s2ZJ2WZm5iUp5+2ornva7zFyTmXXAMuCBYvkiKsX6Pbdl\nZmNmLgVeBg4CjgX+W0Q8A8wH9gLGFesvyMzlzexvGvDrzFxfnEbyM+DoVuQFmJeZGzNzK5Uj5aOA\njzbZ7jbg33diOzOAmzPzbYDM3NDCenOLr7/nz2NyFHBr8bz7gD+18NzGJll+ChxV/BK0Z2Y+XCz/\nMS2PQXP73t5vgDkRcQ7Qs4V1JKldLNmSdnXfp3KEeUCTZfUU/z9GRA+gT5PH6prcbmxyv5H//DmX\n3G4/CQTwleKI6uTMHJOZ75X0Le16FR+saeYGdvx5nA96/a3Z387sa0e2H8d27zszzwP+HhgJ/D4i\n9mp7PElqniVb0i6tOBp7G5Wi/Z4VwJTi9slA7zZs+tSI6FGcpz0WeBG4H/hSRPQGiIjxETHggzYC\nLAA+ERFDI6In8NdUTqNor/nFdvcq8pza5LEVNP/6HwTOioj+Rf4hrdjfb4DPF887Fhjcwno9+POH\nNv8GeCwzNwJ/ioiPF8tn0boxeAsY+N6diDggM+cXH5pcT6VsS1JVObuIJMG1wP9ocv9HwF0R8Sxw\nH207yrySSkHeAzgvM7dGxI1UTmF4KiKCSsH73AdtJDPXRMQ3gV9RORJ+b2be1YY8zW33UuBx4E3g\nmSYPN/v6M/O+iJgMPBkR24CfAxft5C4vA26JiFnFPtdSKb/b2wIcERF/D6yjci45wBlUzm3vT+X0\nm7N29rUC/wHcHhEzqZw3f35EjKMynvOAZ1uxLUnaKZHZ2nfiJElqnYjYDWjIzPqIOBK4PjMnN7Pe\n5szcvfMTSlJ1eSRbktQZ9gduK87x3gacU3IeSepQHsmWJEmSqswPPkqSJElVZsmWJEmSqsySLUmS\nJFWZJVuSJEmqMku2JEmSVGWWbEmSJKnK/h+0iqkn5iqqNwAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(Ms, vfe_lml, label=\"lower\")\n", "plt.plot(Ms, vupper_lml, label=\"upper\")\n", "plt.axhline(full_lml, label=\"full\", alpha=0.3)\n", "plt.xlabel(\"Number of inducing points\")\n", "plt.ylabel(\"LML estimate\")\n", "plt.legend()\n", "plt.title(\"LML bounds for models trained with SGPR\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We see that the lower bound increases as more inducing points are added. Note that the upper bound does _not_ monotonically decrease! This is because as we train the sparse model, we also get better estimates of the hyperparameters. The upper bound will be different for this different setting of the hyperparameters, and is sometimes looser. The upper bound also converges to the true lml slower than the lower bound." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Upper bounds for fixed hyperparameters\n", "Here, we train sparse models with the hyperparameters fixed to the optimal value found previously." ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 62.680716\n", " Number of iterations: 9\n", " Number of functions evaluations: 16\n", "3 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 60.882546\n", " Number of iterations: 13\n", " Number of functions evaluations: 17\n", "4 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.697017\n", " Number of iterations: 9\n", " Number of functions evaluations: 11\n", "5 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.065631\n", " Number of iterations: 10\n", " Number of functions evaluations: 11\n", "6 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.053159\n", " Number of iterations: 12\n", " Number of functions evaluations: 14\n", "7 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023861\n", " Number of iterations: 19\n", " Number of functions evaluations: 22\n", "8 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023374\n", " Number of iterations: 21\n", " Number of functions evaluations: 24\n", "9 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023338\n", " Number of iterations: 20\n", " Number of functions evaluations: 24\n", "10 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023335\n", " Number of iterations: 2\n", " Number of functions evaluations: 5\n", "11 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023334\n", " Number of iterations: 1\n", " Number of functions evaluations: 3\n", "12 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023330\n", " Number of iterations: 5\n", " Number of functions evaluations: 8\n", "13 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023329\n", " Number of iterations: 1\n", " Number of functions evaluations: 3\n", "14 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023328\n", " Number of iterations: 1\n", " Number of functions evaluations: 4\n", "15 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 59.023326\n", " Number of iterations: 1\n", " Number of functions evaluations: 3\n", "16 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023322\n", " Number of iterations: 3\n", " Number of functions evaluations: 6\n", "17 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023321\n", " Number of iterations: 0\n", " Number of functions evaluations: 1\n", "18 INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'\n", " Objective function value: 59.023321\n", " Number of iterations: 0\n", " Number of functions evaluations: 1\n", "19 " ] } ], "source": [ "fMs = np.arange(3, 20, 1)\n", "fvfe_lml = [] # Fixed vfe lml\n", "fvupper_lml = [] # Fixed upper lml\n", "for M in fMs:\n", " Zinit = vfe.feature.Z.read_value()[:M, :].copy()\n", " Zinit = np.vstack((Zinit, X[np.random.permutation(len(X))[:(M - len(Zinit))], :].copy()))\n", " init_params = {p.full_name:p.read_value() for p in vfe.parameters} # TODO: provide convenience function\n", " init_params['SGPR/feature/Z'] = Zinit\n", " vfe = gpflow.models.SGPR(X, Y, gpflow.kernels.RBF(1), Zinit)\n", " vfe.assign(init_params)\n", " vfe.kern.variance.set_trainable(False)\n", " vfe.kern.lengthscales.set_trainable(False)\n", " vfe.likelihood.set_trainable(False)\n", " vfe.compile()\n", " gpflow.train.ScipyOptimizer().minimize(vfe, disp=False)\n", " \n", " fvfe_lml.append(vfe.compute_log_likelihood())\n", " fvupper_lml.append(vfe.compute_upper_bound())\n", " print(\"%i\" % M, end=\" \")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAF3CAYAAABws1edAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xucl3Wd///Hi5PDSQQRkJOgogmoaAPaAfKU1laSW222\nlu3appZbbdvhm7lb1ua3Nm2zvm25lq72yzZd0zRrXdLNU6siICKKhQcUEOUkIHJmXr8/Ph9gwBmY\nGa6Za2Z43G+3z20+1/t6X9f1+lwRPnnP+/O+IjORJEmSVJwuZRcgSZIkdTaGbEmSJKlghmxJkiSp\nYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWDdyi6gCAMH\nDsxRo0aVXYYkSZI6uZkzZy7PzIP21K9ThOxRo0YxY8aMssuQJElSJxcRzzeln9NFJEmSpIIZsiVJ\nkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmS\npIIZsiVJkqSCdSu7gCKs27SFmc+vLLsMSZIkCXAkW5IkSSpcpxjJ7tWjG288ZEDZZUiSJEmAI9mS\nJElS4QzZkiRJUsFKCdkRcXlEPBURcyLi1og4oNo+KiLWR8Ts6uuqMuqTJEmS9kZZI9m/A8Zn5jHA\nn4CL6+17JjMnVF8XllOeJEmS1HKlhOzMnJaZW6qbDwHDy6hDkiRJag3tYU72ecB/1dseXZ0qcm9E\nTC6rKEmSJKmlWm0Jv4i4CxjSwK5LMvO2ap9LgC3ADdV9S4CRmbkiIt4I/CoixmXmmgbOfz5wPsDI\nkSNb4yNIkiRJLdJqITszT9vd/oj4K+DdwKmZmdVjNgIbq+9nRsQzwBHAjAbOfzVwNUBtbW0WWrwk\nSZK0F8paXeQdwBeBMzNzXb32gyKia/X9ocAY4NkyapQkSZJaqqwnPv4A2A/4XUQAPFRdSWQK8PWI\n2AzUARdm5sqSapQkSZJapJSQnZmHN9L+S+CXbVyOJEmSVKj2sLqIJEmS1KkYsiVJkqSCGbIlSZKk\nghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSC\nGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZ\nsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiVJkqSCGbIlSZKkghmy\nJUmSpIIZsiVJkqSCGbIlSZKkgpUSsiPinyJiTkTMjohpETG03r6LI+LpiPhjRJxRRn2SJEnS3ihr\nJPvyzDwmMycAdwBfAYiIscDZwDjgHcAPI6JrSTVKkiRJLVJKyM7MNfU2ewNZfT8V+EVmbszM54Cn\ngUltXZ8kSZK0N7qVdeGIuAw4F1gNnFxtHgY8VK/bomqbJEmS1GG02kh2RNwVEXMbeE0FyMxLMnME\ncAPwty04//kRMSMiZixbtqzo8iVJkqQWa7WR7Mw8rYldbwB+C3wVWAyMqLdveLWtofNfDVwNUFtb\nmw31kSRJkspQ1uoiY+ptTgWeqr6/HTg7IvaLiNHAGGB6W9cnSZIk7Y2y5mR/KyKOBOqA54ELATLz\niYi4CXgS2AJclJlbS6pRkiRJapFSQnZmvm83+y4DLmvDciRJkqRC+cRHSZIkqWCGbEmSJKlghmxJ\nkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmS\nJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIk\nqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSpYIZsSZIkqWCGbEmSJKlghmxJkiSp\nYIZsSZIkqWCGbEmSJKlgpYTsiPiniJgTEbMjYlpEDK22j4qI9dX22RFxVRn1SZIkSXujrJHsyzPz\nmMycANwBfKXevmcyc0L1dWFJ9UmSJEktVkrIzsw19TZ7A1lGHZIkSVJr6FbWhSPiMuBcYDVwcr1d\noyNidrX9HzLz/jLqkyRJklqq1UayI+KuiJjbwGsqQGZekpkjgBuAv60etgQYWZ1G8vfAzyNi/0bO\nf35EzIiIGcuWLWutjyFJkiQ1W2SWO1MjIkYCv83M8Q3suwf4fGbO2N05amtrc8aM3XaRJEmS9lpE\nzMzM2j31K2t1kTH1NqcCT1XbD4qIrtX3hwJjgGfbvkJJkiSp5cqak/2tiDgSqAOeB7atIjIF+HpE\nbK7uuzAzV5ZUoyRJktQipYTszHxfI+2/BH7ZxuVIkiRJhfKJj5IkSVLBDNmSJElSwQzZkiRJUsEM\n2ZIkSVLBDNmSJElSwQzZkiRJUsEM2ZIkSVLBDNmSJElSwQzZkiRJUsEM2ZIkSVLBDNmSJElSwQzZ\nkiRJUsEM2ZIkSVLBDNmSJElSwQzZkiRJUsEM2ZIkSVLBDNmSJElSwQzZkiRJUsEM2ZIkSVLBDNmS\nJElSwQzZkiRJUsEM2ZIkSVLBDNmSJElSwQzZkiRJUsEM2ZIkSVLBDNmSJElSwZocsiOiV2sWIkmS\nJHUWewzZEfHmiHgSeKq6fWxE/LDVK5MkSZI6qKaMZH8XOANYAZCZjwFTWrMoSZIkqSNr0nSRzFy4\nS9PWIi4eEZ+LiIyIgfXaLo6IpyPijxFxRhHXkSRJktpStyb0WRgRbwYyIroDnwHm7e2FI2IEcDrw\nQr22scDZwDhgKHBXRByRmYWEekmSJKktNGUk+0LgImAYsBiYAHyygGt/F/gikPXapgK/yMyNmfkc\n8DQwqYBrSZIkSW2mKSPZR2bmOfUbIuItwB9aetGImAoszszHIqL+rmHAQ/W2F1XbJEmSpA6jKSH7\n/wHHN6FtJxFxFzCkgV2XAF+mMlWkxSLifOB8gJEjR+7NqSRJkqRCNRqyI+JNwJuBgyLi7+vt2h/o\nuqcTZ+ZpjZz3aGA0sG0UezgwKyImUZmOMqJe9+HVtobOfzVwNUBtbW021EeSJEkqw+5GsnsAfap9\n+tZrXwO8v6UXzMzHgUHbtiNiAVCbmcsj4nbg5xHxL1S++DgGmN7Sa0mSJKllNm/ezKJFi9iwYUPZ\npZSipqaG4cOH07179xYd32jIzsx7gXsj4rrMfL6lBTZHZj4RETcBTwJbgItcWUSSJKntLVq0iL59\n+zJq1Ch2+Q5dp5eZrFixgkWLFjF69OgWnaMpc7LXRcTlVJbVq6l38VNadMVdZOaoXbYvAy4r4tyS\nJElqmQ0bNuyTARsgIjjwwANZtmxZi8/RlCX8bqDySPXRwNeABcAjLb6iJEmSOoR9MWBvs7efvSkh\n+8DMvAbYnJn3ZuZ5QCGj2JIkSVJj+vTpU3YJLdaU6SKbqz+XRMS7gBeBAa1XkiRJktR2tm7dSteu\ne1w8r1maMpL9jYjoB3wO+DzwE+CzhVYhSZIkNSIz+cIXvsD48eM5+uijufHGGwG46KKLuP322wE4\n66yzOO+88wC49tprueSSSwD42c9+xqRJk5gwYQIXXHABW7dW1tTo06cPn/vc5zj22GN58MEHC695\njyPZmXlH9e1q4OTCK5AkSVK79rVfP8GTL64p9Jxjh+7PV98zrkl9b7nlFmbPns1jjz3G8uXLmThx\nIlOmTGHy5Mncf//9nHnmmSxevJglS5YAcP/993P22Wczb948brzxRv7whz/QvXt3PvnJT3LDDTdw\n7rnn8tprr3HCCSfwne98p9DPtc0eQ3ZEjAY+BYyq3z8zz2yViiRJkqR6HnjgAT70oQ/RtWtXBg8e\nzNve9jYeeeQRJk+ezJVXXsmTTz7J2LFjeeWVV1iyZAkPPvgg3//+97n++uuZOXMmEydOBGD9+vUM\nGlR5XEvXrl153/ve12o1N2VO9q+Aa4BfA3WtVokkSZLapaaOOLe1YcOGsWrVKu68806mTJnCypUr\nuemmm+jTpw99+/YlM/noRz/KN7/5zdcdW1NTU/g87PqaMid7Q2Z+PzN/X11d5N7qg2okSZKkVjd5\n8mRuvPFGtm7dyrJly7jvvvuYNGkSACeeeCJXXnnl9ukjV1xxBZMnTwbg1FNP5eabb2bp0qUArFy5\nkuefb5NnLDZpJPt7EfFVYBqwcVtjZs5qtaokSZKkqrPOOosHH3yQY489lojg29/+NkOGDAEqAXza\ntGkcfvjhHHLIIaxcuXJ7yB47dizf+MY3OP3006mrq6N79+7867/+K4ccckir1xyZufsOEd8EPgI8\nw47pIlnUEx+LUFtbmzNmzCi7DEmSpE5j3rx5HHXUUWWXUaqG7kFEzMzM2j0d25SR7A8Ah2bmphbW\nJ0mSJO1TmjIney5wQGsXIkmSJHUWTRnJPgB4KiIeYec52S7hJ0mSJDWgKSH7q61ehSRJktSJNOWJ\njy7XJ0mSJDVDoyE7Ih7IzLdGxKtA/SVIgsrqIvu3enWSJElSB9RoyM7Mt1Z/9m27ciRJkqSOb4+r\ni0TE/9eUNkmSJKmjykzq6ur23LGJmrKE304Pq4+IbsAbC6tAkiRJasCCBQsYP3789u0rrriCSy+9\nlJNOOonPfOYzTJgwgfHjxzN9+nQALr30Uj7ykY/wpje9iTFjxvDjH/94+7GXX345EydO5JhjjuGr\nX/3q9vMfeeSRnHvuuYwfP56FCxcWVvvu5mRfDHwZ6BkRa7Y1A5uAqwurQJIkSe3bf30JXnq82HMO\nORre+a0WH75u3Tpmz57Nfffdx3nnncfcuXMBmDNnDg899BCvvfYaxx13HO9617uYO3cu8+fPZ/r0\n6WQmZ555Jvfddx8jR45k/vz5XH/99Zx44olFfTJg93Oyvwl8MyK+mZkXF3pVSZIkaS986EMfAmDK\nlCmsWbOGVatWATB16lR69uxJz549Ofnkk5k+fToPPPAA06ZN47jjjgNg7dq1zJ8/n5EjR3LIIYcU\nHrChaetk3xERvTPztYj4MHA88L3MfL7waiRJktT+7MWI897o1q3bTvOkN2zYsP19ROzUd9t2Q+2Z\nycUXX8wFF1yw074FCxbQu3fvossGmjYn+0fAuog4Fvgc8Azw01apRpIkSaoaPHgwS5cuZcWKFWzc\nuJE77rhj+74bb7wRgAceeIB+/frRr18/AG677TY2bNjAihUruOeee5g4cSJnnHEG1157LWvXrgVg\n8eLFLF26tFVrb8pI9pbMzIiYCvwgM6+JiI+1alWSJEna53Xv3p2vfOUrTJo0iWHDhvGGN7xh+76a\nmhqOO+44Nm/ezLXXXru9/ZhjjuHkk09m+fLl/OM//iNDhw5l6NChzJs3jze96U0A9OnTh5/97Gd0\n7dq11WqPzNx9h4h7gTuB84DJwFLgscw8utWqaqba2tqcMWNG2WVIkiR1GvPmzeOoo44qu4wGnXTS\nSVxxxRXU1tbu1H7ppZfSp08fPv/5zxdynYbuQUTMzMzaRg7ZrinTRT4IbATOy8yXgOHA5S0pVJIk\nSdoX7HG6SGa+FBG/BMZUm5YDt7ZqVZIkSVIj7rnnngbbL7300jatY3ea8sTHjwM3A/9WbRoG/Ko1\ni5IkSZI6sqZMF7kIeAuwBiAz5wODWrMoSZIkqSNrSsjemJmbtm1UH6u++29LSpIkSfuwpoTseyNi\n2+PV3w78J/DrIi4eEZ+LiIyIgdXtURGxPiJmV19XFXEdSZIkqS01JWR/CVgGPA5cAPwW+Ie9vXBE\njABOB17YZdczmTmh+rpwb68jSZKkjun73/8+Rx11FOecc06jffr06QNUnt44fvz4tiptj5qyukgd\n8OPqq0jfBb4I3FbweSVJktQJ/PCHP+Suu+5i+PDhZZfSbE0ZyS5c9emRizPzsQZ2j65OFbk3Iia3\ndW2SJEkq34UXXsizzz7LO9/5Tvr168cVV1yxfd/48eNZsGBBecU1QVMeq94iEXEXMKSBXZcAX6Yy\nVWRXS4CRmbkiIt4I/CoixmXmmgbOfz5wPsDIkSOLK1ySJEk7+dPLr/Lqhs2FnrNvTXeOGNy30f1X\nXXUVd955J7///e/5wQ9+UOi120KLRrIj4oo99cnM0zJz/K4v4FlgNPBYRCyg8gTJWRExJDM3ZuaK\n6vEzgWeAIxo5/9WZWZuZtQcddFBLPoYkSZLUKlo6kv0XQIseCp+Zj1Nvne1q0K7NzOURcRCwMjO3\nRsShVJ4y+WwLa5QkSVIBdjfi3Ba6detGXV3d9u0NGzaUWE3TtHROdhRaxQ5TgDkRMZvKUyYvzMyV\nrXQtSZIkdQCjRo1i1qxZAMyaNYvnnnuu5Ir2rNGR7IgY0NguCgzZmTmq3vtfAr8s6tySJEnq+N73\nvvfx05/+lHHjxnHCCSdwxBENziZuV3Y3XWQmlSc7NhSoi535LkmSJO2i/goi06ZNa7DP2rVrgcpo\n99y5c9uirCZpNGRn5ui2LESSJEnqLHY3XeT43R2YmbOKL0eSJEnq+HY3XWQGMBdYXt2uP20kgVNa\nqyhJkiSpI9tdyP574P3AeuAXwK2ZubZNqpIkSVLpMpOI1lpUrn3LzL06vtEl/DLzysx8K/ApYARw\nd0TcFBET9uqKkiRJavdqampYsWLFXofNjigzWbFiBTU1NS0+xx4fRpOZz0bEbUBP4CNUnsA4u8VX\nlCRJUrs3fPhwFi1axLJly8oupRQ1NTUMHz68xcfv7ouPhwJnA1OBhVSmjPzfzFzf4qtJkiSpQ+je\nvTujR7vYXEvtbiT7aWAOcBuwBhgJfGLbvJzM/JdWr06SJEnqgHYXsr9OZRURgD677Nv3JudIkiRJ\nTbS7h9Fc2ti+iPi7VqlGkiRJ6gQaXV1kD/6+0CokSZKkTqSlIXvfXDBRkiRJaoKWhmznZEuSJEmN\n2N0Sfq/ScJgOKmtmS5IkSWrA7r742LctC5EkSZI6i5ZOF5EkSZLUCEO2JEmSVDBDtiRJklQwQ7Yk\nSZJUMEO2JEmSVDBDtiRJklQwQ7YkSZJUMEO2JEmSVDBDtiRJklQwQ7YkSZJUMEO2JEmSVDBDtiRJ\nklQwQ7YkSZJUsFJCdkRcGhGLI2J29fVn9fZdHBFPR8QfI+KMMuqTJEmS9ka3Eq/93cy8on5DRIwF\nzgbGAUOBuyLiiMzcWkaBkiRJUku0t+kiU4FfZObGzHwOeBqYVHJNkiRJUrOUGbI/FRFzIuLaiOhf\nbRsGLKzXZ1G1TZIkSeowWi1kR8RdETG3gddU4EfAocAEYAnwnRac//yImBERM5YtW1Zw9ZIkSVLL\ntdqc7Mw8rSn9IuLHwB3VzcXAiHq7h1fbGjr/1cDVALW1tdnySiVJkqRilbW6yMH1Ns8C5lbf3w6c\nHRH7RcRoYAwwva3rkyRJkvZGWauLfDsiJgAJLAAuAMjMJyLiJuBJYAtwkSuLSJIkqaMpJWRn5kd2\ns+8y4LI2LEeSJEkqVHtbwk+SJEnq8AzZkiRJUsEM2ZIkSVLBDNmSJElSwQzZe2PGv8OiGVDnAiiS\nJEnaoawl/Dq+DWvgt1+Aus3Q60A4/DQYczocdgr0GlB2dZIkSSqRIbulavaHz/8JnvkfmD8Nnr4L\n5twI0QWGT4Qxb6+E7iHHQETZ1UqSJKkNRWbHfyJ5bW1tzpgxo9wi6rbCi49WAvf838GLsyrtfYZU\nR7nfDoedDDX9yq1TkiRJLRYRMzOzdo/9DNmtZO1SePruSuh+5m7YsBq6dIMRJ+4Y5R50lKPckiRJ\nHYghuz3ZugUWPbJjlPvlxyvt+w/fEbhHT4H9+pRbpyRJknbLkN2erXmxErbnT4Nn74VNr0LXHnDI\nWyqBe8zpcOBhjnJLkiS1M4bsjmLLJlj40I5R7mVPVdr7j94xyj3qrdC9Z7l1SpIkyZDdYb3yPDz9\nu0rgfvZe2LIeutVUppOMOb0SvPuPKrtKSZKkfVJTQ7ZL+LU3/Q+BiX9TeW3eAM8/sGNqyfxplT4D\nj9gRuEe+Gbr1KLdmSZIk7cSR7I5kxTM7AveCB2DrRujRBw49qRK4D3879BtWdpWSJEmdliPZndGB\nh1VeJ14Im16D5+7fMcL91B2VPoPGVQL30R+AIePLrVeSJGkf5Uh2Z5AJy/64I3C/8CDUbYFRk+GE\nC+DIP4MuXcuuUpIkqcPzi4/7snUr4dGfwfQfw+oXoN9ImPQ3cPy50LN/2dVJkiR1WIZsVR71/sff\nwsP/Bgvuh+694JgPVka3Bx1VdnWSJEkdjnOyVZkictR7Kq+X5sLDV8Fj/wEz/x1Gvw1O/ERllRKn\nkkiSJBXKkex9zWsrYNb18MhPYM3iyprbk86HCedAzwPKrk6SJKldc7qIdm/rFnjq15WpJC88CN17\nw4S/rATug44ouzpJkqR2yeki2r2u3WDcWZXXi7MrYXvW9fDIj+GwUytTSQ47Fbp0KbtSSZKkDseR\nbO2wdhnMvK4ylWTtSzDgsMqXJI/9ENTsX3Z1kiRJpXO6iFpuyyaYd3vli5KLHoEefeG4cypTSQ48\nrOzqJEmSSmPIVjEWzYTp/wZzb6k84GbM6ZXR7cNOgYiyq5MkSWpThmwV69WXYMa/w4xr4LVlMPCI\nStg+5mzYr0/Z1UmSJLUJQ7Zax5aN8MSt8NCPYMls2K8fHP8RmPg3MGB02dVJkiS1KkO2WldmZb72\nw1fBk7dVni555DvhhAth9BSnkkiSpE6pXS/hFxGXAh8HllWbvpyZv42IUcA84I/V9ocy88I2L1B7\nFgEjJlVea16ER66pPEnyj7+Fg46qTiX5IPToVXalkiRJba6UkexqyF6bmVfs0j4KuCMzxzfnfI5k\ntxObN8DcX8LDP4KXHoeaA+CNH61MJTlgZNnVSZIk7bWmjmT7pBEVp3tNZam/C+6Hv/4vOPRt8L//\nD753LNz4YVjwQGWaiSRJUidXZsj+VETMiYhrI6J/vfbRETE7Iu6NiMmlVaeWi4BD3gx/8VP4zBx4\n86crAfu6d8FPToWn7zZsS5KkTq3VpotExF3AkAZ2XQI8BCwHEvgn4ODMPC8i9gP6ZOaKiHgj8Ctg\nXGauaeD85wPnA4wcOfKNzz//fKt8DhVk0zqY8wu4/19g9UIY+WY45R9g1FvKrkySJKnJOszqIrub\nhx0R9wCfz8zdTrh2TnYHsmUjzPop3Hc5rH0ZDj25EraH7/HPqiRJUuna9ZzsiDi43uZZwNxq+0ER\n0bX6/lBgDPBs21eoVtNtP5j0cfj0bDj9G/DSnMoUkp9/EJY8VnZ1kiRJhShrTva3I+LxiJgDnAx8\ntto+BZgTEbOBm4ELM3NlSTWqNfXoBW/+FHzmMTjlH+GFB+HfpsBN58LSp8quTpIkaa+UPl2kCE4X\n6QTWr4IH/xUe+iFseg2O/gCc9CU48LCyK5MkSdquXU8XkV6n5wFwyiWV1Uje8mmY92v4wUS47W9h\n1QtlVydJktQshmy1L70PhLd/vTKNZNLHYc6N8P3j4TefhzVLyq5OkiSpSQzZap/6DoZ3/jN8+lE4\n7sOVR7Z/fwL89yWwdlnZ1UmSJO2WIVvtW7/h8J4r4W9nwLg/r8zZ/t6xcPfXYZ3fiZUkSe2TIVsd\nw4DRcNaP4KLpcOQ74P7vVML2Pf8MG173rCJJkqRSGbLVsQwcA++/Fj7xvzB6Ctzzf+F7x8AD362s\nSiJJktQOGLLVMQ0eB2ffAB//PQyrhbsurYxsP/Qj2Lyh7OokSdI+zpCtjm3Y8fDhm+G8aTDoKLjz\nS/D94+CRa2DLprKrkyRJ+yhDtjqHkSfAR38N594OB4yA3/w9/OCN8OgNsHVL2dVJkqR9jCFbncuh\nb4Pz/hvOuRl6DoDbPgk/PAEevxnq6squTpIk7SMM2ep8ImDM2+H8e+CDN0DXHvDLj8FVb6k8STKz\n7AolSVInZ8hW5xUBR70bLvwDvO8a2LoZbvwwXH0S/GmaYVuSJLUaQ7Y6vy5d4Oj3wycfgvf+CNa/\nAj//AFxzOjx7b9nVSZKkTqhb2QVIbaZrN5jwlzD+/TD7Z3DfFfDTM2HUZDjmg5WnS/YbAf2GQfee\nZVcrSZI6sMhO8Cvz2tranDFjRtllqKPZvAFmXld5euRrS3fe1+vASujef3g1fA+vhO9+Iyrv+wyG\nLl1LKVuSJJUnImZmZu2e+jmSrX1X9xo48UKY+DewZhGsXgSrF8PqhZX3axbDK8/Bgvth4y6Pbu/S\nDfoOrQbvagjfv14I7zcMag6ozAuXJEn7HEO21LUb9B9VeTVmw+pqAF+0SyBfBAunwxO/grrNOx/T\no0+98D1859f+wyqv7jWt+ckkSVJJDNlSU9T0q7wGj214f11dZcrJ6kU7v7YF8pcef/2UFIDeg3aM\nhu8//PVhvPegyhc3JUlSh2LIlorQpQv0HVJ5DW9kmtbmDZUpKGsWvz6ML/sTPP0/sPm1nY/p0QcO\nekMl3A8at+Nn7wNb/zNJkqQWM2RLbaV7DRx4WOXVkEzYsGrHNJTVC2H5fFj6JMy7A2b9dEffPoNh\n0NjKa3D150FvgB692uazSJKk3TJkS+1FBPTsX3kNGb/zvkxYuxSWPgEvP1kJ3i8/ATOugS0btp0A\nBhz6+lHvAaNdCUWSpDZmyJY6ggjoO7jyOuyUHe11W2Hlc/XCd/XnvDuA6vKc3WrgoCPrBe+xMHhc\nZTTc1U8kSWoVhmypI+vSFQYeXnmNnbqjfdM6WP7HnUe9n7kbHvv5jj49B1TC9qCxMOio6vujYL++\nbf85JEnqZAzZUmfUoxcMPa7yqu+1FTtPOVn6JMy+ATat3dHngJE7j3oPGgsDx0DX7m37GSRJ6sAM\n2dK+pPeBMHpK5bVNXR2sfmHn6SZLn4T50yC3Vvp06Q4Dj9gRvPseXBlFjy7Vn113+dmlkX1ddt7e\n3mc359l+vl33OdVFktR+GbK1T1q3aQuLXlnPolfWseiV9azduIXMHfuzupG5fWZz9f2Odqju29Z3\np/bdH1//x+6ute18dVl5X5e5/Zjt2wl19fpnZnW7sp/cw/FUj88DyZxCMpm67tB1yEaGbF7EyM0L\nGLH5OUY1vrw8AAARaUlEQVSuWsDI5fdw0OP/ube3vxB1BHV02f7K6EIdQVIJ31nvPUTlcze2HVHt\nT739O7Yr/Ro6R71jGj3Htnraq+b9YyWb2b9yTEvs+TpNraVJ12/iP9pa8vn3Zd4vtaYu77mSw499\nS9llNMqQrU5pw+atLF61nkWvrGfhykqQXlgN1ItWrmPFa5sKvd62/z4HENWNqLcv2NEhXndMNH58\nQJeo7O8SsX1/ZbtybJfY0Va/X5fKhXfajm19qQ4qV48nquep129r9GBh90NZ3OMwHqpXR6+6tfTN\ntXTJrZV4m5WYG9TtaKvX3oWtRGa99q31+u/os+N9HVHv/Y7z1+vTwHGVOLsjZu8UkbOx+Fx/u9IW\nWT1m23a9nzsdl/WO2SVKN3SNdimbV1dL4lLLPvuej2nqeQuM4e33f8f2qpl/vqTm2r97j7JL2C1D\ntjqkzVvrWLJqQzU4r2Physqo9MLq6PTLazbu1L9712DYAT0ZMaAXp48bzPD+vRgxoBfD+/dkeP+e\n7F9TmW/cpOBb7RdNHPmSJEn7HkO22qWtdclLazawaOWO4LxwZWU0evEr61myej119QZJunYJDu5X\nw/D+PZky5qDtAXrbz8F9a+jSxVAsSZLahiFbpairS5at3bjzKPTK9SxaVfn54qr1bKmXoiNgcN8a\nRgzoyaTRAxjRvyfDtwXp/r04uF8N3bp2KfETSZIk7VBayI6ITwEXAVuB32TmF6vtFwMfq7Z/OjP/\nu6waVYxlr27kkQUreWTBSp5Z9tr2Lxtu2lK3U7+BffZjeP+eHDviAN59zMHVKR09Gd6/F0MPqGG/\nbj61UJIkdQylhOyIOBmYChybmRsjYlC1fSxwNjAOGArcFRFHZG5bR0ztXWbywsp1TH9uZTVYv8Jz\ny18DoKZ7F8YM6suRg/ty2lGDK6PR1SA97IBe9OxhiJYkSZ1DWSPZnwC+lZkbATJzabV9KvCLavtz\nEfE0MAl4sJwytSd1dckfX36VRxas3B6st33psF/P7kwc1Z+zJ45g4ugBjB/ajx7dnNIhSZI6v7JC\n9hHA5Ii4DNgAfD4zHwGGAQ/V67eo2vY6EXE+cD7AyJEjW7dabbdpSx2PL17F9Ode4ZEFK5mxYCVr\nNmwB4OB+NZww+kAmjh7ApFEDGDOoj182lCRJ+6RWC9kRcRcwpIFdl1SvOwA4EZgI3BQRhzbn/Jl5\nNXA1QG1trYtxtpK1G7cw6/lXto9Uz164io3VudSHHtSbPzv6YCaNHsDEUQMY3r+ny9pJkiTRiiE7\nM09rbF9EfAK4JSuPupseEXXAQGAxMKJe1+HVNrWRFWs38siCHaH6ySVr2FqXdAkYN7Qf55xwCJNG\n96d21AAG9tmv7HIlSZLapbKmi/wKOBn4fUQcAfQAlgO3Az+PiH+h8sXHMcD0kmrs9DKTRa+s377y\nx/TnKqt/AOzXrQsTRhzAJ086jImjBnD8If3ps58rPkqSJDVFWanpWuDaiJgLbAI+Wh3VfiIibgKe\nBLYAF7mySHHq6pKnl63l4edW8kj1S4pLVm8AoG9NN2oP6c/73ziCSaP7M35YP5fMkyRJaqFSQnZm\nbgI+3Mi+y4DL2raizmnz1jrmLl5dHaV+hRnPr2TVus0ADOq73/YvKE4cNYAjh/Slq19SlCRJKoS/\n/+9kXlixjl/PeZE/PL2cR19YxfrNlV8EjB7Ym9PHDmbiqAFMGj2AkQN6+SVFSZKkVmLI7gRWrdvE\nbx5fwq2zFjPj+VcAGHvw/nxw4ggmjhrAxNH9GdS3puQqJUmS9h2G7A5q45at/P6pZdz66CJ+/9Qy\nNm2t4/BBffjCGUfy3uOGMeyAnmWXKEmStM8yZHcgmcnM51/hlkcX85s5S1i9fjMD++zHh088hD8/\nfhjjhu7vFBBJkqR2wJDdATy7bC2/enQxt85ezMKV66np3oUzxg3hrOOG8dbDB9Ktq48qlyRJak8M\n2e3UirUbuWPOEm55dDGPLVxFBLzlsIH83alHcMb4Ia5ZLUmS1I6Z1NqRDZu3cte8l7l11mLu/dMy\nttQlbxjSly//2Rs489hhDOnnlxclSZI6AkN2yerqkoefW8mtjy7ivx5/iVc3bmHw/vvxsbeO5r3H\nDeOog/cvu0RJkiQ1kyG7JPNffpVbH13MbbNfZPGq9fTu0ZV3jD+YPz9+GCceeqAPhpEkSerADNlt\naOmrG7h99ov8avZi5i5eQ9cuweQxA/niO47k9LFD6NnDx5hLkiR1BobsVrZu0xamPfEytz66mPvn\nL6Mu4ehh/fjKu8fynmOHclDf/couUZIkSQUzZLeCrXXJ/z6znFsfXcx/z32J1zZtZdgBPfnESYdx\n1nHDOHxQ37JLlCRJUisyZBdo3pI11XnWi3l5zUb61nTjPccO5b3HDWPSqAF0cZ61JEnSPsGQvZde\nWr2B22Yv5tZHF/PUS6/SrUtw0pGD+Mq7h3HqUYOo6e48a0mSpH2NIbuFVq/fzCdvmMn/PrOCTDhu\n5AF8feo43n3MUAb07lF2eZIkSSqRIbuF9q/pRrcuXfjUKWM467hhjB7Yu+ySJEmS1E4YslsoIrj+\nvElllyFJkqR2qEvZBUiSJEmdjSFbkiRJKpghW5IkSSqYIVuSJEkqmCFbkiRJKpghW5IkSSqYIVuS\nJEkqmCFbkiRJKpghW5IkSSqYIVuSJEkqmCFbkiRJKpghW5IkSSqYIVuSJEkqWGRm2TXstYhYBjxf\ndh3tzEBgedlFdCDer+bxfjWP96t5vF/N4/1qHu9X83i/Xu+QzDxoT506RcjW60XEjMysLbuOjsL7\n1Tzer+bxfjWP96t5vF/N4/1qHu9XyzldRJIkSSqYIVuSJEkqmCG787q67AI6GO9X83i/msf71Tze\nr+bxfjWP96t5vF8t5JxsSZIkqWCOZEuSJEkFM2R3QhHRNSIejYg7yq6lvYuIAyLi5oh4KiLmRcSb\nyq6pPYuIz0bEExExNyL+IyJqyq6pvYmIayNiaUTMrdc2ICJ+FxHzqz/7l1lje9LI/bq8+v/JORFx\na0QcUGaN7UlD96vevs9FREbEwDJqa48au18R8anqn7EnIuLbZdXX3jTy/8cJEfFQRMyOiBkRManM\nGjsSQ3bn9BlgXtlFdBDfA+7MzDcAx+J9a1REDAM+DdRm5nigK3B2uVW1S9cB79il7UvA3Zk5Bri7\nuq2K63j9/fodMD4zjwH+BFzc1kW1Y9fx+vtFRIwATgdeaOuC2rnr2OV+RcTJwFTg2MwcB1xRQl3t\n1XW8/s/Xt4GvZeYE4CvVbTWBIbuTiYjhwLuAn5RdS3sXEf2AKcA1AJm5KTNXlVtVu9cN6BkR3YBe\nwIsl19PuZOZ9wMpdmqcC11ffXw+8t02Lascaul+ZOS0zt1Q3HwKGt3lh7VQjf74Avgt8EfCLVvU0\ncr8+AXwrMzdW+yxt88LaqUbuVwL7V9/3w7/3m8yQ3flcSeUv2rqyC+kARgPLgH+vTq/5SUT0Lruo\n9iozF1MZ8XkBWAKszsxp5VbVYQzOzCXV9y8Bg8sspoM5D/ivsotozyJiKrA4Mx8ru5YO4ghgckQ8\nHBH3RsTEsgtq5/4OuDwiFlL5b4C/WWoiQ3YnEhHvBpZm5syya+kgugHHAz/KzOOA1/DX+I2qziOe\nSuUfJ0OB3hHx4XKr6niysqSTo41NEBGXAFuAG8qupb2KiF7Al6n8Gl9N0w0YAJwIfAG4KSKi3JLa\ntU8An83MEcBnqf72V3tmyO5c3gKcGRELgF8Ap0TEz8otqV1bBCzKzIer2zdTCd1q2GnAc5m5LDM3\nA7cAby65po7i5Yg4GKD6019P70FE/BXwbuCcdK3Z3TmMyj98H6v+3T8cmBURQ0qtqn1bBNySFdOp\n/ObXL4s27qNU/r4H+E/ALz42kSG7E8nMizNzeGaOovKFtP/JTEcaG5GZLwELI+LIatOpwJMlltTe\nvQCcGBG9qqM+p+IXRZvqdir/oaL687YSa2n3IuIdVKa9nZmZ68qupz3LzMczc1Bmjqr+3b8IOL76\n95sa9ivgZICIOALoASwvtaL27UXgbdX3pwDzS6ylQ+lWdgFSyT4F3BARPYBngb8uuZ52KzMfjoib\ngVlUfoX/KD4J7HUi4j+Ak4CBEbEI+CrwLSq/kv4Y8DzwF+VV2L40cr8uBvYDflf9Lf5DmXlhaUW2\nIw3dr8z01/eNaOTP17XAtdVl6jYBH/W3JRWN3K+PA9+rfuF9A3B+eRV2LD7xUZIkSSqY00UkSZKk\nghmyJUmSpIIZsiVJkqSCGbIlSZKkghmyJUmSpIIZsiXtkyIiI+I79bY/HxGXFnTu6yLi/UWcaw/X\n+UBEzIuI3+/SPrS63GJzzvVXEfGDFtbx9Yg4rSXH7q2mXDsiTooIH5wkqU25TrakfdVG4M8j4puZ\n2W4eRBER3TJzSxO7fwz4eGY+UL8xM18EWj3k17teaY/0buK1TwLWAv/butVI0g6OZEvaV22h8jCd\nz+66Y9eR6IhYW/15UkTcGxG3RcSzEfGtiDgnIqZHxOMRcVi905wWETMi4k8R8e7q8V0j4vKIeCQi\n5kTEBfXOe39E3E4DTx2NiA9Vzz83Iv652vYV4K3ANRFx+S79R1UftLFthPqWiLgzIuZHxLfr9fvr\nan3Tgbfs6fNX3/+fai2PRcS3du0fEQsi4msRMava7w3V9oMi4ncR8URE/CQino+I1z3KOiLWRsR3\nq/3ujoiDqu0TIuKh6n27NSL6N+XaETEKuBD4bETMjojJ1d8AzK1+hvt2rUGSimDIlrQv+1fgnIjo\n14xjjqUS2o4CPgIckZmTgJ9QeYLoNqOAScC7gKsioobKyPPqzJwITAQ+HhGjq/2PBz6TmUfUv1hE\nDAX+mcrjjCcAEyPivZn5dWAGcE5mfmEPNU8APggcDXwwIkZExMHA16iE67cCY/f0wSPincBU4ITM\nPBb4diNdl2fm8cCPgM9X274K/E9mjgNuBkY2cmxvYEa1373V4wB+CvyfzDwGeLxe+26vnZkLgKuA\n72bmhMy8H/gKcEb1M5y5p88tSS1hyJa0z8rMNVTC26ebcdgjmbkkMzcCzwDTqu2PUwnW29yUmXWZ\nOR94FngDcDpwbkTMBh4GDgTGVPtPz8znGrjeROCezFxWnUZyAzClGfUC3J2ZqzNzA5WR8kOAE+qd\ndxNwYxPOcxrw75m5DiAzVzbS75bqz5nsuCdvBX5RPe5O4JVGjq2rV8vPgLdW/xF0QGbeW22/nsbv\nQUPX3tUfgOsi4uNA10b6SNJeMWRL2tddSWWEuXe9ti1U/36MiC5Aj3r7NtZ7X1dvu46dv+eSu1wn\ngQA+VR1RnZCZozNzW0h/ba8+xe7Vr3kre/4+zu4+f3Ou15Rr7cmu93Gvr52ZFwL/AIwAZkbEgS0v\nT5IaZsiWtE+rjsbeRCVob7MAeGP1/ZlA9xac+gMR0aU6T/tQ4I/AfwOfiIjuABFxRET03t1JgOnA\n2yJiYER0BT5EZRrF3nq4et4Dq/V8oN6+BTT8+X8H/HVE9KrWP6AZ1/sD8BfV404H+jfSrws7vrT5\nl8ADmbkaeCUiJlfbP0Lz7sGrQN9tGxFxWGY+XP3S5DIqYVuSCuXqIpIE3wH+tt72j4HbIuIx4E5a\nNsr8ApWAvD9wYWZuiIifUJnCMCsigkrAe+/uTpKZSyLiS8DvqYyE/yYzb2tBPQ2d91LgQWAVMLve\n7gY/f2beGRETgBkRsQn4LfDlJl7ya8B/RMRHqtd8iUr43dVrwKSI+AdgKZW55AAfpTK3vReV6Td/\n3dTPCvwauDkiplKZN//ZiBhD5X7eDTzWjHNJUpNEZnN/EydJUvNExH7A1szcEhFvAn6UmRMa6Lc2\nM/u0fYWSVCxHsiVJbWEkcFN1jvcm4OMl1yNJrcqRbEmSJKlgfvFRkiRJKpghW5IkSSqYIVuSJEkq\nmCFbkiRJKpghW5IkSSqYIVuSJEkq2P8PZ8pL7C1MH7oAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(fMs, fvfe_lml, label=\"lower\")\n", "plt.plot(fMs, fvupper_lml, label=\"upper\")\n", "plt.axhline(full_lml, label=\"full\", alpha=0.3)\n", "plt.xlabel(\"Number of inducing points\")\n", "plt.ylabel(\"LML estimate\")\n", "plt.legend()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now, as the hyperparameters are fixed, the bound _does_ monotonically decrease. We chose the optimal hyperparameters here, but the picture should be the same for any hyperparameter setting. This shows that we increasingly get a better estimate of the marginal likelihood as we add more inducing points." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## A tight estimate bound does not imply a converged model" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Optimization terminated with:\n", " Message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'\n", " Objective function value: 62.487090\n", " Number of iterations: 46\n", " Number of functions evaluations: 57\n", "Lower bound: -62.487090\n", "Upper bound: -62.484641\n" ] } ], "source": [ "vfe = gpflow.models.SGPR(X, Y, gpflow.kernels.RBF(1), X[None, 0, :].copy())\n", "vfe.compile()\n", "gpflow.train.ScipyOptimizer().minimize(vfe)\n", "print(\"Lower bound: %f\" % vfe.compute_log_likelihood())\n", "print(\"Upper bound: %f\" % vfe.compute_upper_bound())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In this case we show that for the hyperparameter setting, the bound is very tight. However, this does _not_ imply that we have enough inducing points, but simply that we have correctly identified the marginal likelihood for this particular hyperparameter setting. In this specific case, where we used a single inducing point, the model collapses to not using the GP at all (lengthscale is really long to model only the mean). The rest of the variance is explained by noise. This GP can be perfectly approximated with a single inducing point." ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
classpriortransformtrainableshapefixed_shapevalue
SGPR/kern/varianceParameterNone+veTrue()True0.10775260845160224
SGPR/kern/lengthscalesParameterNone+veTrue()True1014.3847358198036
SGPR/likelihood/varianceParameterNone+veTrue()True0.682431983343512
SGPR/feature/ZParameterNone(none)True(1, 1)True[[2.65740317504]]
\n", "
" ], "text/plain": [ " class prior transform trainable shape \\\n", "SGPR/kern/variance Parameter None +ve True () \n", "SGPR/kern/lengthscales Parameter None +ve True () \n", "SGPR/likelihood/variance Parameter None +ve True () \n", "SGPR/feature/Z Parameter None (none) True (1, 1) \n", "\n", " fixed_shape value \n", "SGPR/kern/variance True 0.10775260845160224 \n", "SGPR/kern/lengthscales True 1014.3847358198036 \n", "SGPR/likelihood/variance True 0.682431983343512 \n", "SGPR/feature/Z True [[2.65740317504]] " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vfe.as_pandas_table()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This can be diagnosed by showing that there are other hyperparameter settings with higher upper bounds. This indicates that there may be better hyperparameter settings, but we cannot identify them due to the lack of inducing points. An example of this can be seen in the previous section." ] } ], "metadata": { "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 1 }