diff --git a/.gitignore b/.gitignore index 80d0d0288..687c47162 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ handyspark \.ipynb handyspark1\.ipynb examples/new-api-sandbox\.ipynb + +examples/new-api-optimus-jars\.ipynb diff --git a/.pypirc b/.pypirc deleted file mode 100644 index 2004732ad..000000000 --- a/.pypirc +++ /dev/null @@ -1,29 +0,0 @@ -[distutils] -index-servers= - testpypi - pypi - -[testpypi] -repository : https://test.pypi.org/legacy/ -username : argenisleon - - -[pypi] -repository : https://upload.pypi.org/legacy/ -username : argenisleon - - - -# Reference https://dzone.com/articles/executable-package-pip-install -# Run to upload the package to pypi -# To build the package -# python setup.py bdist_wheel - -# To the main pypi -# python -m twine upload dist/* -r pypi --config-file .pypirc - -# To pypi test -# python -m twine upload dist/* -r testpypi --config-file .pypirc - -# Test package installation -# python -m pip install dist/* \ No newline at end of file diff --git a/.python-version b/.python-version deleted file mode 100644 index 5a958026d..000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.5 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a2a3474aa..000000000 --- a/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -FROM jupyter/scipy-notebook - -USER root - -RUN pip install --no-cache-dir optimuspyspark -U -RUN git clone https://github.com/ironmussa/Optimus-examples.git /home/jovyan/work -RUN chmod -R 777 /home/jovyan/work - -# Spark dependencies -ENV APACHE_SPARK_VERSION 2.3.1 -ENV HADOOP_VERSION 2.7 - -RUN apt-get -y update && \ - apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -RUN cd /tmp && \ - wget -q http://d3kbcqa49mib13.cloudfront.net/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \ - echo "7a186a2a007b2dfd880571f7214a7d329c972510a460a8bdbef9f7f2a891019343c020f74b496a61e5aa42bc9e9a79cc99defe5cb3bf8b6f49c07e01b259bc6b *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local && \ - rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark - -# Mesos dependencies -RUN . /etc/os-release && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF && \ - DISTRO=$ID && \ - CODENAME=$VERSION_CODENAME && \ - echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" > /etc/apt/sources.list.d/mesosphere.list && \ - apt-get -y update && \ - apt-get --no-install-recommends -y --force-yes install mesos=1.2\* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Spark and Mesos config -ENV SPARK_HOME /usr/local/spark -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.4-src.zip -ENV MESOS_NATIVE_LIBRARY /usr/local/lib/libmesos.so -ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info - -USER $NB_USER \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 854c7fc7f..d6828fc5f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include optimus/templates/* include optimus/profiler/templates/* include optimus/css/* +include optimus/jars/* diff --git a/README.md b/README.md index 9a397a092..b9ea9341a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + [![Logo Optimus](images/logoOptimus.png)](https://hioptimus.com) @@ -225,12 +226,18 @@ Let's load a "big" dataset df = op.load.csv("https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/Meteorite_Landings.csv").h_repartition() ``` +### Numeric + ```python -op.profiler.run(df, "name", infer=False) +op.profiler.run(df, "mass (g)", infer=False) ``` +![](images/profiler_numeric.png) + ```python +op.profiler.run(df, "name", infer=False) ``` + ![](images/profiler.png) For dates data types Optimus can give you extra data @@ -238,8 +245,6 @@ For dates data types Optimus can give you extra data op.profiler.run(df, "year", infer=True) ``` -```python -``` ![](images/profiler1.png) ## Plots @@ -251,15 +256,29 @@ df = df.rows.drop_na(["age","fare"]) ``` ```python -df.plot.scatter(["fare", "age"], buckets=30) +df.plot.hist("fare", output="image", path="images/hist.png") +``` + +```python +df.plot.frequency("age", output="image", path="images/frequency.png") ``` ```python -df.plot.box("age") +df.plot.scatter(["fare", "age"], buckets=30, output="image", path="images/scatter.png") +``` + +```python +df.plot.box("age", output="image", path="images/box.png") ``` ```python df.plot.correlation(["age","fare","survived"]) ``` +### Using other plotting libraries + + +Optimus has a tiny API so you can use any plotting library. For example, you can use df.cols.scatter(), df.cols.frequency(), df.cols.boxplot() or df.cols.hist() to output a JSON that you can process to adapt the data to any plotting library. + + ## Outliers @@ -282,9 +301,9 @@ df.outliers.iqr("age").drop().table() -``` +```python df.outliers.z_score("age", threshold=2).drop() -df.outliers.modified_z_score("age", threshold = 2 ).drop() +df.outliers.modified_z_score("age", threshold = 2).drop() df.outliers.mad("age", threshold = 2).drop() ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index a86eec3ee..f5c351c6e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -60,7 +60,7 @@ # The short X.Y version. version = '2.2' # The full version, including alpha/beta/rc tags. -release = '2.2.51' +release = '2.2.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/examples/10_min_from_spark_to_pandas_with_optimus.ipynb b/examples/10_min_from_spark_to_pandas_with_optimus.ipynb index cc17fa2dc..87a47e9d6 100644 --- a/examples/10_min_from_spark_to_pandas_with_optimus.ipynb +++ b/examples/10_min_from_spark_to_pandas_with_optimus.ipynb @@ -12,11 +12,19 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import sys\n", "sys.path.append(\"..\")" ] }, @@ -44,12 +52,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from optimus import Optimus" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from optimus import Optimus\n", - "op= Optimus(master=\"local\")" + "op = Optimus(master=\"local\")" ] }, { @@ -439,14 +455,14 @@ " \"attributes\"\n", " ],\n", " [\n", - " \n", - " (\"Optim'us\", 28.0, \"Leader\", 10, 4.3, [\"Inochi\", \"Convoy\"], \"19.442735,-99.201111\",[8.5344, 4300.0]),\n", - " (\"bumbl#ebéé \", 17.5, \"Espionage\", 7, 2.0, [\"Bumble\",\"Goldback\"], \"10.642707,-71.612534\",[5.334, 2000.0]),\n", - " (\"ironhide&\", 26.0, \"Security\", 7, 4.0, [\"Roadbuster\"], \"37.789563,-122.400356\",[7.9248, 4000.0]),\n", - " (\"Jazz\",13.0, \"First Lieutenant\", 8, 1.8, [\"Meister\"], \"33.670666,-117.841553\",[3.9624, 1800.0]),\n", - " (\"Megatron\",None, \"None\", None, 5.7, [\"Megatron\"], None,[None,5700.0]),\n", - " (\"Metroplex_)^$\",300.0 , \"Battle Station\", 8, None, [\"Metroflex\"],None,[91.44, None]),\n", - " \n", + "\n", + " (\"Optim'us\", 28.0, \"Leader\", 10, 4.3, [\"Inochi\", \"Convoy\"], \"19.442735,-99.201111\", [8.5344, 4300.0]),\n", + " (\"bumbl#ebéé \", 17.5, \"Espionage\", 7, 2.0, [\"Bumble\", \"Goldback\"], \"10.642707,-71.612534\", [5.334, 2000.0]),\n", + " (\"ironhide&\", 26.0, \"Security\", 7, 4.0, [\"Roadbuster\"], \"37.789563,-122.400356\", [7.9248, 4000.0]),\n", + " (\"Jazz\", 13.0, \"First Lieutenant\", 8, 1.8, [\"Meister\"], \"33.670666,-117.841553\", [3.9624, 1800.0]),\n", + " (\"Megatron\", None, \"None\", None, 5.7, [\"Megatron\"], None, [None, 5700.0]),\n", + " (\"Metroplex_)^$\", 300.0, \"Battle Station\", 8, None, [\"Metroflex\"], None, [91.44, None]),\n", + "\n", " ]).h_repartition(1)\n", "df.table()" ] @@ -685,9 +701,9 @@ " (\"bumbl#ebéé \", 17.5, \"Espionage\", 7),\n", " (\"Optim'us\", 28.0, \"Leader\", 10),\n", " (\"ironhide&\", 26.0, \"Security\", 7),\n", - " (\"Jazz\",13.0, \"First Lieutenant\", 8),\n", - " (\"Megatron\",None, \"None\", None),\n", - " \n", + " (\"Jazz\", 13.0, \"First Lieutenant\", 8),\n", + " (\"Megatron\", None, \"None\", None),\n", + "\n", " ])\n", "df.table()" ] @@ -920,9 +936,9 @@ " (\"bumbl#ebéé \", 17.5, \"Espionage\", 7),\n", " (\"Optim'us\", 28.0, \"Leader\", 10),\n", " (\"ironhide&\", 26.0, \"Security\", 7),\n", - " (\"Jazz\",13.0, \"First Lieutenant\", 8),\n", - " (\"Megatron\",None, \"None\", None),\n", - " \n", + " (\"Jazz\", 13.0, \"First Lieutenant\", 8),\n", + " (\"Megatron\", None, \"None\", None),\n", + "\n", " ])\n", "df.table()" ] @@ -936,186 +952,21 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "
Viewing 3 of 3 rows / 4 columns
\n", - "
1 partition(s)
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "
names
\n", - "
1 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
height
\n", - "
2 (double)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
function
\n", - "
3 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
rank
\n", - "
4 (bigint)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - "
\n", - " bumbl#ebéé⸱⸱\n", - " \n", - " 17.5\n", - " \n", - " Espionage\n", - " \n", - " 7\n", - "
\n", - " Optim'us\n", - " \n", - " 28.0\n", - " \n", - " Leader\n", - " \n", - " 10\n", - "
\n", - " ironhide&\n", - " \n", - " 26.0\n", - " \n", - " Security\n", - " \n", - " 7\n", - "
\n", - "\n", - "
Viewing 3 of 3 rows / 4 columns
\n", - "
1 partition(s)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", - "import numpy as np\n", "\n", "data = [(\"bumbl#ebéé \", 17.5, \"Espionage\", 7),\n", - " (\"Optim'us\", 28.0, \"Leader\", 10),\n", - " (\"ironhide&\", 26.0, \"Security\", 7)]\n", + " (\"Optim'us\", 28.0, \"Leader\", 10),\n", + " (\"ironhide&\", 26.0, \"Security\", 7)]\n", "labels = [\"names\", \"height\", \"function\", \"rank\"]\n", "\n", "# Create pandas dataframe\n", "pdf = pd.DataFrame.from_records(data, columns=labels)\n", "\n", - "df = op.create.df(pdf = pdf)\n", + "df = op.create.df(pdf=pdf)\n", "df.table()" ] }, @@ -2198,7 +2049,7 @@ } ], "source": [ - "df.rows.select(df[\"rank\"]>7).table()" + "df.rows.select(df[\"rank\"] > 7).table()" ] }, { @@ -2378,7 +2229,7 @@ } ], "source": [ - "df.rows.is_in(\"rank\",[7, 10]).table()" + "df.rows.is_in(\"rank\", [7, 10]).table()" ] }, { @@ -2778,7 +2629,7 @@ } ], "source": [ - "df.cols.append(\"Affiliation\",\"Autobot\").table()" + "df.cols.append(\"Affiliation\", \"Autobot\").table()" ] }, { @@ -2961,7 +2812,7 @@ } ], "source": [ - "df.rows.drop_na(\"*\",how='any').table()" + "df.rows.drop_na(\"*\", how='any').table()" ] }, { @@ -3141,7 +2992,7 @@ } ], "source": [ - "df.cols.fill_na(\"*\",\"N//A\").table()" + "df.cols.fill_na(\"*\", \"N//A\").table()" ] }, { @@ -3563,7 +3414,8 @@ "def func(value, args):\n", " return value + 1\n", "\n", - "df.cols.apply(\"height\",func,\"float\").table()" + "\n", + "df.cols.apply(\"height\", func, \"float\").table()" ] }, { @@ -3776,8 +3628,8 @@ } ], "source": [ - "df\\\n", - " .cols.lower(\"names\")\\\n", + "df \\\n", + " .cols.lower(\"names\") \\\n", " .cols.upper(\"function\").table()" ] }, @@ -4047,11 +3899,10 @@ " (\"Autobot\"),\n", " (\"Autobot\"),\n", " (\"Decepticons\"),\n", - " \n", - " \n", + "\n", " ]).h_repartition(1)\n", "\n", - "op.concat([df,df_new], \"columns\").table()" + "op.concat([df, df_new], \"columns\").table()" ] }, { @@ -4252,10 +4103,10 @@ " \"rank\",\n", " ],\n", " [\n", - " (\"Grimlock\", 22.9, \"Dinobot Commander\", 9), \n", + " (\"Grimlock\", 22.9, \"Dinobot Commander\", 9),\n", " ]).h_repartition(1)\n", "\n", - "op.concat([df,df_new], \"rows\").table()\n" + "op.concat([df, df_new], \"rows\").table()" ] }, { @@ -4443,7 +4294,7 @@ } ], "source": [ - "df_melt = df.melt(id_vars=[\"names\"], value_vars=[\"height\", \"function\",\"rank\"])\n", + "df_melt = df.melt(id_vars=[\"names\"], value_vars=[\"height\", \"function\", \"rank\"])\n", "df.table()" ] }, @@ -4617,7 +4468,7 @@ } ], "source": [ - "df_melt.pivot(\"names\",\"variable\",\"value\").table()" + "df_melt.pivot(\"names\", \"variable\", \"value\").table()" ] }, { @@ -4653,7 +4504,7 @@ } ], "source": [ - "df.plot.hist(\"height\",10)" + "df.plot.hist(\"height\", 10)" ] }, { @@ -4703,7 +4554,7 @@ } ], "source": [ - "df.plot.frequency(\"*\",10)" + "df.plot.frequency(\"*\", 10)" ] }, { @@ -5560,7 +5411,7 @@ } ], "source": [ - "op.profiler.run(df, \"height\",infer=True)" + "op.profiler.run(df, \"height\", infer=True)" ] }, { @@ -5903,7 +5754,7 @@ } ], "source": [ - "df_csv =op.load.url(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.csv\").limit(5)\n", + "df_csv = op.load.url(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.csv\").limit(5)\n", "df_csv.table()" ] }, @@ -6247,7 +6098,8 @@ } ], "source": [ - "df_json =op.load.url(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.json\", \"json\").limit(5)\n", + "df_json = op.load.url(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.json\",\n", + " \"json\").limit(5)\n", "df_json.table()" ] }, @@ -6257,7 +6109,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_csv.save.csv(\"test.csv\") " + "df_csv.save.csv(\"test.csv\")" ] }, { @@ -6470,18 +6322,21 @@ "source": [ "import requests\n", "\n", + "\n", "def func_request(params):\n", " # You can use here whatever header or auth info you need to send. \n", " # For more information see the requests library\n", - " url= \"https://jsonplaceholder.typicode.com/todos/\" + str(params[\"rank\"])\n", + " url = \"https://jsonplaceholder.typicode.com/todos/\" + str(params[\"rank\"])\n", "\n", " return requests.get(url)\n", "\n", + "\n", "def func_response(response):\n", " # Here you can parse de response\n", " return response[\"title\"]\n", "\n", - "df_result = op.enrich(df, func_request= func_request, func_response= func_response)" + "\n", + "df_result = op.enrich(df, func_request=func_request, func_response=func_response)" ] }, { @@ -6678,13 +6533,6 @@ "source": [ "df_result.table()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/data/foo.tsv b/examples/data/foo.tsv new file mode 100644 index 000000000..2f05c3ffd --- /dev/null +++ b/examples/data/foo.tsv @@ -0,0 +1,6 @@ +Sepal length Sepal width Petal length Petal width Species +5.1 3.5 1.4 0.2 I. setosa +4.9 3.0 1.4 0.2 I. setosa +4.7 3.2 1.3 0.2 I. setosa +4.6 3.1 1.5 0.2 I. setosa +5.0 3.6 1.4 0.2 I. setosa \ No newline at end of file diff --git a/examples/hacking_optimus.py b/examples/hacking_optimus.py index c90321dc2..8a0f2e04d 100644 --- a/examples/hacking_optimus.py +++ b/examples/hacking_optimus.py @@ -22,37 +22,39 @@ # If you are in the example folder. This is the way to find optimus import sys + sys.path.append("..") # Create Optimus from optimus import Optimus + op = Optimus() # # Go to optimus/create.py. In 'def data_frame()' write print("Hello World"). Now run the cell below and you should see "Hello World" bellow the cell # + -from pyspark.sql.types import StructType, StructField, StringType, BooleanType, IntegerType, ArrayType +from pyspark.sql.types import StringType, IntegerType, ArrayType df = op.create.df( - [ - ("words", "str", True), - ("num", "int", True), - ("animals", "str", True), - ("thing", StringType(), True), - ("two strings", StringType(), True), - ("filter", StringType(), True), - ("num 2", "string", True), - ("col_array", ArrayType(StringType()), True), - ("col_int", ArrayType(IntegerType()), True) - - ] -, -[ - (" I like fish ", 1, "dog", "housé", "cat-car", "a","1",["baby", "sorry"],[1,2,3]), - (" zombies", 2, "cat", "tv", "dog-tv", "b","2",["baby 1", "sorry 1"],[3,4]), - ("simpsons cat lady", 2, "frog", "table","eagle-tv-plus","1","3", ["baby 2", "sorry 2"], [5,6,7]), - (None, 3, "eagle", "glass", "lion-pc", "c","4", ["baby 3", "sorry 3"] ,[7,8]) - ]) - -#df.table() + [ + ("words", "str", True), + ("num", "int", True), + ("animals", "str", True), + ("thing", StringType(), True), + ("two strings", StringType(), True), + ("filter", StringType(), True), + ("num 2", "string", True), + ("col_array", ArrayType(StringType()), True), + ("col_int", ArrayType(IntegerType()), True) + + ] + , + [ + (" I like fish ", 1, "dog", "housé", "cat-car", "a", "1", ["baby", "sorry"], [1, 2, 3]), + (" zombies", 2, "cat", "tv", "dog-tv", "b", "2", ["baby 1", "sorry 1"], [3, 4]), + ("simpsons cat lady", 2, "frog", "table", "eagle-tv-plus", "1", "3", ["baby 2", "sorry 2"], [5, 6, 7]), + (None, 3, "eagle", "glass", "lion-pc", "c", "4", ["baby 3", "sorry 3"], [7, 8]) + ]) + +# df.table() diff --git a/examples/new-api-livy.ipynb b/examples/new-api-livy.ipynb new file mode 100644 index 000000000..b9c14162a --- /dev/null +++ b/examples/new-api-livy.ipynb @@ -0,0 +1,319 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(\"..\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n", + " from collections import Callable\n", + "\n", + " You are using PySparkling of version 2.4.10, but your PySpark is of\n", + " version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n" + ] + } + ], + "source": [ + "from optimus.livy import Livy" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "HOST = 'http://46.101.172.155:8998'\n", + "livy = Livy(HOST)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 52, 'name': None, 'appId': None, 'owner': None, 'proxyUser': None, 'state': 'starting', 'kind': 'pyspark', 'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None}, 'log': ['stdout: ', '\\nstderr: ']}\n" + ] + } + ], + "source": [ + "# Create session\n", + "livy.start()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 52,\n", + " 'name': None,\n", + " 'appId': None,\n", + " 'owner': None,\n", + " 'proxyUser': None,\n", + " 'state': 'starting',\n", + " 'kind': 'pyspark',\n", + " 'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None},\n", + " 'log': ['stdout: ', '\\nstderr: ']}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get session info\n", + "livy.session()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 2, 'code': '1 + 1', 'state': 'available', 'output': {'status': 'ok', 'execution_count': 2, 'data': {'text/plain': '2'}}, 'progress': 1.0}\n" + ] + } + ], + "source": [ + "data = {'code': '1 + 1'}\n", + "\n", + "response = livy.execute(data)\n", + "print(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 33, 'state': 'running', 'output': None, 'progress': 0.0}\n" + ] + } + ], + "source": [ + "import textwrap, pprint\n", + "code = \"\"\"\n", + " from optimus import Optimus \n", + " op = Optimus(spark)\n", + " df = op.load.csv(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.csv\")\n", + " \"\"\"\n", + "\n", + "response = livy.submit(code)\n", + "print(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 34, 'state': 'waiting', 'output': None, 'progress': 0.0}\n" + ] + } + ], + "source": [ + "import textwrap, pprint\n", + "code = \"\"\" \n", + " import json\n", + " json.dumps(df.to_json())\n", + " \"\"\"\n", + "\n", + "response = livy.submit(code)\n", + "print(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [], + "source": [ + "a = livy.result()" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'[{\"id\": 1, \"product\": \"Cake\", \"firstName\": \"Luis\", \"price\": 10, \"lastName\": \"Alvarez$$%!\", \"billingId\": 123, \"dummyCol\": \"never\", \"birth\": \"1980/07/07\"}, {\"id\": 2, \"product\": \"piza\", \"firstName\": \"Andr\\\\u00e9\", \"price\": 8, \"lastName\": \"Amp\\\\u00e8re\", \"billingId\": 423, \"dummyCol\": \"gonna\", \"birth\": \"1950/07/08\"}, {\"id\": 3, \"product\": \"pizza\", \"firstName\": \"NiELS\", \"price\": 8, \"lastName\": \"B\\\\u00f6hr//((%%\", \"billingId\": 551, \"dummyCol\": \"give\", \"birth\": \"1990/07/09\"}, {\"id\": 4, \"product\": \"pizza\", \"firstName\": \"PAUL\", \"price\": 8, \"lastName\": \"dirac$\", \"billingId\": 521, \"dummyCol\": \"you\", \"birth\": \"1954/07/10\"}, {\"id\": 5, \"product\": \"pizza\", \"firstName\": \"Albert\", \"price\": 8, \"lastName\": \"Einstein\", \"billingId\": 634, \"dummyCol\": \"up\", \"birth\": \"1990/07/11\"}, {\"id\": 6, \"product\": \"arepa\", \"firstName\": \"Galileo\", \"price\": 5, \"lastName\": \" GALiLEI\", \"billingId\": 672, \"dummyCol\": \"never\", \"birth\": \"1930/08/12\"}, {\"id\": 7, \"product\": \"taco\", \"firstName\": \"CaRL\", \"price\": 3, \"lastName\": \"Ga%%%uss\", \"billingId\": 323, \"dummyCol\": \"gonna\", \"birth\": \"1970/07/13\"}, {\"id\": 8, \"product\": \"taaaccoo\", \"firstName\": \"David\", \"price\": 3, \"lastName\": \"H$$$ilbert\", \"billingId\": 624, \"dummyCol\": \"let\", \"birth\": \"1950/07/14\"}, {\"id\": 9, \"product\": \"taco\", \"firstName\": \"Johannes\", \"price\": 3, \"lastName\": \"KEPLER\", \"billingId\": 735, \"dummyCol\": \"you\", \"birth\": \"1920/04/22\"}, {\"id\": 10, \"product\": \"taco\", \"firstName\": \"JaMES\", \"price\": 3, \"lastName\": \"M$$ax%%well\", \"billingId\": 875, \"dummyCol\": \"down\", \"birth\": \"1923/03/12\"}, {\"id\": 11, \"product\": \"pasta\", \"firstName\": \"Isaac\", \"price\": 9, \"lastName\": \"Newton\", \"billingId\": 992, \"dummyCol\": \"never \", \"birth\": \"1999/02/15\"}, {\"id\": 12, \"product\": \"pasta\", \"firstName\": \"Emmy%%\", \"price\": 9, \"lastName\": \"N\\\\u00f6ether$\", \"billingId\": 234, \"dummyCol\": \"gonna\", \"birth\": \"1993/12/08\"}, {\"id\": 13, \"product\": \"hamburguer\", \"firstName\": \"Max!!!\", \"price\": 4, \"lastName\": \"Planck!!!\", \"billingId\": 111, \"dummyCol\": \"run \", \"birth\": \"1994/01/04\"}, {\"id\": 14, \"product\": \"pizzza\", \"firstName\": \"Fred\", \"price\": 8, \"lastName\": \"Hoy&&&le\", \"billingId\": 553, \"dummyCol\": \"around\", \"birth\": \"1997/06/27\"}, {\"id\": 15, \"product\": \"pizza\", \"firstName\": \"((( Heinrich )))))\", \"price\": 8, \"lastName\": \"Hertz\", \"billingId\": 116, \"dummyCol\": \"and\", \"birth\": \"1956/11/30\"}, {\"id\": 16, \"product\": \"BEER\", \"firstName\": \"William\", \"price\": 2, \"lastName\": \"Gilbert###\", \"billingId\": 886, \"dummyCol\": \"desert\", \"birth\": \"1958/03/26\"}, {\"id\": 17, \"product\": \"Rice\", \"firstName\": \"Marie\", \"price\": 1, \"lastName\": \"CURIE\", \"billingId\": 912, \"dummyCol\": \"you\", \"birth\": \"2000/03/22\"}, {\"id\": 18, \"product\": \"110790\", \"firstName\": \"Arthur\", \"price\": 5, \"lastName\": \"COM%%%pton\", \"billingId\": 812, \"dummyCol\": \"#\", \"birth\": \"1899/01/01\"}, {\"id\": 19, \"product\": \"null\", \"firstName\": \"JAMES\", \"price\": 10, \"lastName\": \"Chadwick\", \"billingId\": 467, \"dummyCol\": \"#\", \"birth\": \"1921/05/03\"}]'\n" + ] + }, + { + "ename": "JSONDecodeError", + "evalue": "Expecting value: line 1 column 1 (char 0)", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mJSONDecodeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32m~\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m 346\u001b[0m \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 347\u001b[0m parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 348\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 349\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 350\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m 335\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 336\u001b[0m \"\"\"\n\u001b[1;32m--> 337\u001b[1;33m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 338\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 339\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m 353\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 354\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 355\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 356\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)" + ] + } + ], + "source": [ + "import json\n", + "print(a)\n", + "json.loads(a)" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{}" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "{}" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "http://46.101.172.155:8998/sessions/46\n" + ] + } + ], + "source": [ + "livy.delete_session(46)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 52,\n", + " 'name': None,\n", + " 'appId': None,\n", + " 'owner': None,\n", + " 'proxyUser': None,\n", + " 'state': 'idle',\n", + " 'kind': 'pyspark',\n", + " 'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None},\n", + " 'log': ['19/06/04 03:52:01 INFO Executor: Fetching spark://10.19.0.5:42304/jars/livy-repl_2.11-0.6.0-incubating.jar with timestamp 1559619584198',\n", + " '19/06/04 03:52:01 INFO Utils: Fetching spark://10.19.0.5:42304/jars/livy-repl_2.11-0.6.0-incubating.jar to /tmp/spark-e906068a-b29b-4594-aef1-58a2d37bde15/userFiles-d5869831-80e5-4da1-aa9c-37245d12c7f2/fetchFileTemp3930129784218452512.tmp',\n", + " '19/06/04 03:52:01 INFO Executor: Adding file:/tmp/spark-e906068a-b29b-4594-aef1-58a2d37bde15/userFiles-d5869831-80e5-4da1-aa9c-37245d12c7f2/livy-repl_2.11-0.6.0-incubating.jar to class loader',\n", + " '19/06/04 03:52:02 INFO PythonRunner: Times: total = 898, boot = 676, init = 55, finish = 167',\n", + " '19/06/04 03:52:02 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1401 bytes result sent to driver',\n", + " '19/06/04 03:52:02 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1515 ms on localhost (executor driver) (1/1)',\n", + " '19/06/04 03:52:02 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool ',\n", + " '19/06/04 03:52:02 INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 49371',\n", + " '19/06/04 03:52:02 INFO DAGScheduler: ResultStage 0 (reduce at :8) finished in 1.819 s',\n", + " '19/06/04 03:52:02 INFO DAGScheduler: Job 0 finished: reduce at :8, took 1.931369 s']}]" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "livy.sessions()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/new-api-optimus.ipynb b/examples/new-api-optimus.ipynb index d25d6452f..56da54f35 100644 --- a/examples/new-api-optimus.ipynb +++ b/examples/new-api-optimus.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -31,23 +31,35 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# Run if you clone Optimus and want to hack and test from the notebook\n", "\n", - "# %load_ext autoreload\n", - "# %autoreload 2\n", - "# import sys\n", - "# sys.path.append(\"..\")" + "%load_ext autoreload\n", + "%autoreload 2\n", + "import sys\n", + "sys.path.append(\"..\")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n", + " from collections import Callable\n", + "\n", + " You are using PySparkling of version 2.4.10, but your PySpark is of\n", + " version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n" + ] + } + ], "source": [ "from optimus import Optimus" ] @@ -61,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -71,10 +83,10 @@ "INFO:optimus:Just check that Spark and all necessary environments vars are present...\n", "INFO:optimus:-----\n", "INFO:optimus:SPARK_HOME=C:\\opt\\spark\\spark-2.3.1-bin-hadoop2.7\n", - "INFO:optimus:HADOOP_HOME=C:\\opt\\spark\\spark-2.3.1-bin-hadoop2.7\n", + "INFO:optimus:HADOOP_HOME=C:\\opt\\hadoop-2.7.7\n", "INFO:optimus:PYSPARK_PYTHON=C:\\Users\\argenisleon\\Anaconda3\\python.exe\n", "INFO:optimus:PYSPARK_DRIVER_PYTHON=jupyter\n", - "INFO:optimus:PYSPARK_SUBMIT_ARGS=--conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", + "INFO:optimus:PYSPARK_SUBMIT_ARGS=--packages com.databricks:spark-avro_2.11:4.0.0 --conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", "INFO:optimus:JAVA_HOME=C:\\java\n", "INFO:optimus:Pyarrow Installed\n", "INFO:optimus:-----\n", @@ -112,9 +124,7 @@ " }\n", "\n", " .optimus_table td {\n", - " padding: 2px;\n", - " border-left: 1px solid #cccccc;\n", - " border-right: 1px solid #cccccc;\n", + " border: 0px;\n", " }\n", "\n", " .optimus_table tr:nth-child(even) {\n", @@ -165,9 +175,17 @@ " .panel_profiler td {\n", " padding:0.2em\n", " }\n", - " .none{\n", + " .none, .true{\n", " color:#0000ff\n", " }\n", + " .optimus_table th {\n", + " font-family:sans-serif;\n", + " }\n", + "\n", + " .info_items{\n", + " font-family:sans-serif;\n", + " font-size:10px;\n", + " }\n", "" ], "text/plain": [ @@ -192,95 +210,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "op = Optimus(packages=\"datastax:spark-cassandra-connector:1.6.1-s_2.10\")" ] @@ -294,7 +226,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -302,20 +234,20 @@ "text/html": [ "\n", "
\n", - "

SparkSession - in-memory

\n", + "

SparkSession - hive

\n", " \n", "
\n", "

SparkContext

\n", "\n", - "

Spark UI

\n", + "

Spark UI

\n", "\n", "
\n", "
Version
\n", "
v2.3.1
\n", "
Master
\n", - "
local[*]
\n", + "
local
\n", "
AppName
\n", - "
pyspark-shell
\n", + "
optimus
\n", "
\n", "
\n", " \n", @@ -323,10 +255,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -344,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -354,24 +286,24 @@ "
\n", "

SparkContext

\n", "\n", - "

Spark UI

\n", + "

Spark UI

\n", "\n", "
\n", "
Version
\n", "
v2.3.1
\n", "
Master
\n", - "
local[*]
\n", + "
local
\n", "
AppName
\n", - "
pyspark-shell
\n", + "
optimus
\n", "
\n", "
\n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -389,16 +321,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['datastax:spark-cassandra-connector:1.6.1-s_2.10']" + "['com.databricks:spark-avro_2.11:4.0.0']" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -416,7 +348,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -602,67 +534,99 @@ " \n", " \n", " \n", - "
Optim'us
\n", + "
Optim'us\n", + "
\n", " \n", " \n", " \n", - "
28
\n", + "
28\n", + "
\n", " \n", " \n", " \n", - "
Leader
\n", + "
Leader\n", + "
\n", " \n", " \n", " \n", - "
10
\n", + "
10\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
4.300000190734863
\n", + "
4.300000190734863\n", + "
\n", " \n", " \n", " \n", - "
['Inochi',⸱'Convoy']
\n", + "
['Inochi',⋅'Convoy']\n", + "
\n", " \n", " \n", " \n", - "
19.442735,-99.201111
\n", + "
19.442735,-99.201111\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2016/09/10
\n", + "
2016/09/10\n", + "
\n", " \n", " \n", " \n", - "
[8.53439998626709,⸱4300.0]
\n", + "
[8.53439998626709,⋅4300.0]\n", + "
\n", " \n", " \n", " \n", - "
2016-09-10
\n", + "
2016-09-10\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'Leader')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -670,67 +634,99 @@ " \n", " \n", " \n", - "
bumbl#ebéé⸱⸱
\n", + "
bumbl#ebéé⋅⋅\n", + "
\n", " \n", " \n", " \n", - "
17
\n", + "
17\n", + "
\n", " \n", " \n", " \n", - "
Espionage
\n", + "
Espionage\n", + "
\n", " \n", " \n", " \n", - "
7
\n", + "
7\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
2.0
\n", + "
2.0\n", + "
\n", " \n", " \n", " \n", - "
['Bumble',⸱'Goldback']
\n", + "
['Bumble',⋅'Goldback']\n", + "
\n", " \n", " \n", " \n", - "
10.642707,-71.612534
\n", + "
10.642707,-71.612534\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2015/08/10
\n", + "
2015/08/10\n", + "
\n", " \n", " \n", " \n", - "
[5.334000110626221,⸱2000.0]
\n", + "
[5.334000110626221,⋅2000.0]\n", + "
\n", " \n", " \n", " \n", - "
2015-08-10
\n", + "
2015-08-10\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'Espionage')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -738,67 +734,99 @@ " \n", " \n", " \n", - "
ironhide&
\n", + "
ironhide&\n", + "
\n", " \n", " \n", " \n", - "
26
\n", + "
26\n", + "
\n", " \n", " \n", " \n", - "
Security
\n", + "
Security\n", + "
\n", " \n", " \n", " \n", - "
7
\n", + "
7\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
4.0
\n", + "
4.0\n", + "
\n", " \n", " \n", " \n", - "
['Roadbuster']
\n", + "
['Roadbuster']\n", + "
\n", " \n", " \n", " \n", - "
37.789563,-122.400356
\n", + "
37.789563,-122.400356\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2014/07/10
\n", + "
2014/07/10\n", + "
\n", " \n", " \n", " \n", - "
[7.924799919128418,⸱4000.0]
\n", + "
[7.924799919128418,⋅4000.0]\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24
\n", + "
2014-06-24\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'Security')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -806,67 +834,99 @@ " \n", " \n", " \n", - "
Jazz
\n", + "
Jazz\n", + "
\n", " \n", " \n", " \n", - "
13
\n", + "
13\n", + "
\n", " \n", " \n", " \n", - "
First⸱Lieutenant
\n", + "
First⋅Lieutenant\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
1.7999999523162842
\n", + "
1.7999999523162842\n", + "
\n", " \n", " \n", " \n", - "
['Meister']
\n", + "
['Meister']\n", + "
\n", " \n", " \n", " \n", - "
33.670666,-117.841553
\n", + "
33.670666,-117.841553\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2013/06/10
\n", + "
2013/06/10\n", + "
\n", " \n", " \n", " \n", - "
[3.962399959564209,⸱1800.0]
\n", + "
[3.962399959564209,⋅1800.0]\n", + "
\n", " \n", " \n", " \n", - "
2013-06-24
\n", + "
2013-06-24\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'First⸱Lieutenant')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -874,67 +934,99 @@ " \n", " \n", " \n", - "
Megatron
\n", + "
Megatron\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
10
\n", + "
10\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
5.699999809265137
\n", + "
5.699999809265137\n", + "
\n", " \n", " \n", " \n", - "
['Megatron']
\n", + "
['Megatron']\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2012/05/10
\n", + "
2012/05/10\n", + "
\n", " \n", " \n", " \n", - "
[None,⸱5700.0]
\n", + "
[None,⋅5700.0]\n", + "
\n", " \n", " \n", " \n", - "
2012-05-10
\n", + "
2012-05-10\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'None')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -942,67 +1034,99 @@ " \n", " \n", " \n", - "
Metroplex_)^$
\n", + "
Metroplex_)^$\n", + "
\n", " \n", " \n", " \n", - "
300
\n", + "
300\n", + "
\n", " \n", " \n", " \n", - "
Battle⸱Station
\n", + "
Battle⋅Station\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
5000000
\n", + "
5000000\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
['Metroflex']
\n", + "
['Metroflex']\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
1980/04/10
\n", + "
1980/04/10\n", + "
\n", " \n", " \n", " \n", - "
2011/04/10
\n", + "
2011/04/10\n", + "
\n", " \n", " \n", " \n", - "
[91.44000244140625,⸱None]
\n", + "
[91.44000244140625,⋅None]\n", + "
\n", " \n", " \n", " \n", - "
2011-04-10
\n", + "
2011-04-10\n", + "
\n", " \n", " \n", " \n", - "
2014-06-24⸱00:00:00
\n", + "
2014-06-24⋅00:00:00\n", + "
\n", " \n", " \n", " \n", - "
True
\n", + "
True\n", + "
\n", " \n", " \n", " \n", - "
bytearray(b'Battle⸱Station')
\n", + "
None\n", + "
\n", " \n", " \n", " \n", - "
None
\n", + "
None\n", + "
\n", " \n", " \n", " \n", @@ -1078,7 +1202,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1102,10 +1226,33 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "scrolled": true }, + "outputs": [], + "source": [ + "op.append([df,df], like=\"rows\").table()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## IO Operations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load from file" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, "outputs": [ { "data": { @@ -1115,7 +1262,7 @@ "\n", "\n", "\n", - "
Viewing 12 of 12 rows / 16 columns
\n", + "
Viewing 5 of 19 rows / 8 columns
\n", "
1 partition(s)
\n", "\n", "\n", @@ -1123,8 +1270,8 @@ " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + "
firstName
\n", + "
2 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", " \n", + "\n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -1970,67 +1409,103 @@ " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2038,67 +1513,103 @@ " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2107,7 +1618,7 @@ "
\n", - "
names
\n", - "
1 (string)
\n", + "
id
\n", + "
1 (int)
\n", "
\n", " \n", " nullable\n", @@ -1133,836 +1280,128 @@ "
\n", - "
height(ft)
\n", - "
2 (int)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
function
\n", - "
3 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
rank
\n", - "
4 (int)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
age
\n", - "
5 (int)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
weight(t)
\n", - "
6 (float)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
japanese name
\n", - "
7 (array<string>)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
last position seen
\n", - "
8 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
date arrival
\n", - "
9 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
last date seen
\n", - "
10 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
attributes
\n", - "
11 (array<float>)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
DateType
\n", - "
12 (date)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
Tiemstamp
\n", - "
13 (timestamp)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
Cybertronian
\n", - "
14 (boolean)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
function(binary)
\n", - "
15 (binary)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
NullType
\n", - "
16 (null)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - "
\n", - "
Optim'us
\n", - " 		\n", - "
28
\n", - " 		\n", - "
Leader
\n", - " 		\n", - "
10
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
4.300000190734863
\n", - " 		\n", - "
['Inochi',⸱'Convoy']
\n", - " 		\n", - "
19.442735,-99.201111
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2016/09/10
\n", - " 		\n", - "
[8.53439998626709,⸱4300.0]
\n", - " 		\n", - "
2016-09-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Leader')
\n", - " 		\n", - "
None
\n", - "
\n", - "
bumbl#ebéé⸱⸱
\n", - " 		\n", - "
17
\n", - " 		\n", - "
Espionage
\n", - " 		\n", - "
7
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
2.0
\n", - " 		\n", - "
['Bumble',⸱'Goldback']
\n", - " 		\n", - "
10.642707,-71.612534
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2015/08/10
\n", - " 		\n", - "
[5.334000110626221,⸱2000.0]
\n", - " 		\n", - "
2015-08-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Espionage')
\n", - " 		\n", - "
None
\n", - "
\n", - "
ironhide&
\n", - " 		\n", - "
26
\n", - " 		\n", - "
Security
\n", - " 		\n", - "
7
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
4.0
\n", - " 		\n", - "
['Roadbuster']
\n", - " 		\n", - "
37.789563,-122.400356
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2014/07/10
\n", - " 		\n", - "
[7.924799919128418,⸱4000.0]
\n", - " 		\n", - "
2014-06-24
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Security')
\n", - " 		\n", - "
None
\n", - "
\n", - "
Jazz
\n", - " 		\n", - "
13
\n", - " 		\n", - "
First⸱Lieutenant
\n", - " 		\n", - "
8
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
1.7999999523162842
\n", - " 		\n", - "
['Meister']
\n", - " 		\n", - "
33.670666,-117.841553
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2013/06/10
\n", - " 		\n", - "
[3.962399959564209,⸱1800.0]
\n", - " 		\n", - "
2013-06-24
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'First⸱Lieutenant')
\n", - " 		\n", - "
None
\n", - "
\n", - "
Megatron
\n", - " 		\n", - "
None
\n", - " 		\n", - "
None
\n", - " 		\n", - "
10
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
5.699999809265137
\n", - " 		\n", - "
['Megatron']
\n", - " 		\n", - "
None
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2012/05/10
\n", - " 		\n", - "
[None,⸱5700.0]
\n", - " 		\n", - "
2012-05-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'None')
\n", - " 		\n", - "
None
\n", - "
\n", - "
Metroplex_)^$
\n", - " 		\n", - "
300
\n", - " 		\n", - "
Battle⸱Station
\n", - " 		\n", - "
8
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
None
\n", - " 		\n", - "
['Metroflex']
\n", - " 		\n", - "
None
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2011/04/10
\n", - " 		\n", - "
[91.44000244140625,⸱None]
\n", - " 		\n", - "
2011-04-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Battle⸱Station')
\n", - " 		\n", - "
None
\n", - "
\n", - "
Optim'us
\n", - " 		\n", - "
28
\n", - " 		\n", - "
Leader
\n", - " 		\n", - "
10
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
4.300000190734863
\n", - " 		\n", - "
['Inochi',⸱'Convoy']
\n", - " 		\n", - "
19.442735,-99.201111
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2016/09/10
\n", - " 		\n", - "
[8.53439998626709,⸱4300.0]
\n", - " 		\n", - "
2016-09-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Leader')
\n", - " 		\n", - "
None
\n", - "
\n", - "
bumbl#ebéé⸱⸱
\n", - " 		\n", - "
17
\n", - " 		\n", - "
Espionage
\n", - " 		\n", - "
7
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
2.0
\n", - " 		\n", - "
['Bumble',⸱'Goldback']
\n", - " 		\n", - "
10.642707,-71.612534
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2015/08/10
\n", - " 		\n", - "
[5.334000110626221,⸱2000.0]
\n", - " 		\n", - "
2015-08-10
\n", - " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", - "
True
\n", - " 		\n", - "
bytearray(b'Espionage')
\n", - " 		\n", - "
None
\n", - "
\n", - "
ironhide&
\n", - " 		\n", - "
26
\n", - " 		\n", - "
Security
\n", - " 		\n", - "
7
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
4.0
\n", - " 		\n", - "
['Roadbuster']
\n", - " 		\n", - "
37.789563,-122.400356
\n", - " 		\n", - "
1980/04/10
\n", - " 		\n", - "
2014/07/10
\n", - " 		\n", - "
[7.924799919128418,⸱4000.0]
\n", - " 		\n", + "
lastName
\n", + "
3 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", - "
2014-06-24
\n", - " 		\n", + "
billingId
\n", + "
4 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", - "
2014-06-24⸱00:00:00
\n", - " 		\n", + "
product
\n", + "
5 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", - "
True
\n", - " 		\n", + "
price
\n", + "
6 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", - "
bytearray(b'Security')
\n", - " 		\n", + "
birth
\n", + "
7 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", - "
None
\n", - " 		\n", + "
dummyCol
\n", + "
8 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", - "
Jazz
\n", - " 		\n", - "
13
\n", - " 		\n", - "
First⸱Lieutenant
\n", - " 		\n", - "
8
\n", - " 		\n", - "
5000000
\n", - " 		\n", - "
1.7999999523162842
\n", - " 		\n", - "
['Meister']
\n", - " 		\n", - "
33.670666,-117.841553
\n", - " 		\n", - "
1980/04/10
\n", + "
1\n", + "
\n", " 		\n", - "
2013/06/10
\n", + "
Luis\n", + "
\n", " 		\n", - "
[3.962399959564209,⸱1800.0]
\n", + "
Alvarez$$%!\n", + "
\n", " 		\n", - "
2013-06-24
\n", + "
123\n", + "
\n", " 		\n", - "
2014-06-24⸱00:00:00
\n", + "
Cake\n", + "
\n", " 		\n", - "
True
\n", + "
10\n", + "
\n", " 		\n", - "
bytearray(b'First⸱Lieutenant')
\n", + "
1980/07/07\n", + "
\n", " 		\n", - "
None
\n", + "
never\n", + "
\n", "
\n", - "
Megatron
\n", + "
2\n", + "
\n", " 		\n", - "
None
\n", + "
André\n", + "
\n", " 		\n", - "
None
\n", + "
Ampère\n", + "
\n", " 		\n", - "
10
\n", + "
423\n", + "
\n", " 		\n", - "
5000000
\n", + "
piza\n", + "
\n", " 		\n", - "
5.699999809265137
\n", + "
8\n", + "
\n", " 		\n", - "
['Megatron']
\n", + "
1950/07/08\n", + "
\n", " 		\n", - "
None
\n", + "
gonna\n", + "
\n", "
\n", - "
1980/04/10
\n", + "
3\n", + "
\n", " 		\n", - "
2012/05/10
\n", + "
NiELS\n", + "
\n", " 		\n", - "
[None,⸱5700.0]
\n", + "
Böhr//((%%\n", + "
\n", " 		\n", - "
2012-05-10
\n", + "
551\n", + "
\n", " 		\n", - "
2014-06-24⸱00:00:00
\n", + "
pizza\n", + "
\n", " 		\n", - "
True
\n", + "
8\n", + "
\n", " 		\n", - "
bytearray(b'None')
\n", + "
1990/07/09\n", + "
\n", " 		\n", - "
None
\n", + "
give\n", + "
\n", "
\n", - "
Metroplex_)^$
\n", + "
4\n", + "
\n", " 		\n", - "
300
\n", + "
PAUL\n", + "
\n", " 		\n", - "
Battle⸱Station
\n", + "
dirac$\n", + "
\n", " 		\n", - "
8
\n", + "
521\n", + "
\n", " 		\n", - "
5000000
\n", + "
pizza\n", + "
\n", " 		\n", - "
None
\n", + "
8\n", + "
\n", " 		\n", - "
['Metroflex']
\n", + "
1954/07/10\n", + "
\n", " 		\n", - "
None
\n", + "
you\n", + "
\n", "
\n", - "
1980/04/10
\n", + "
5\n", + "
\n", " 		\n", - "
2011/04/10
\n", + "
Albert\n", + "
\n", " 		\n", - "
[91.44000244140625,⸱None]
\n", + "
Einstein\n", + "
\n", " 		\n", - "
2011-04-10
\n", + "
634\n", + "
\n", " 		\n", - "
2014-06-24⸱00:00:00
\n", + "
pizza\n", + "
\n", " 		\n", - "
True
\n", + "
8\n", + "
\n", " 		\n", - "
bytearray(b'Battle⸱Station')
\n", + "
1990/07/11\n", + "
\n", " 		\n", - "
None
\n", + "
up\n", + "
\n", "
\n", "\n", "\n", - "
Viewing 12 of 12 rows / 16 columns
\n", + "
Viewing 5 of 19 rows / 8 columns
\n", "
1 partition(s)
\n" ], "text/plain": [ @@ -2119,26 +1630,12 @@ } ], "source": [ - "op.append([df,df], like=\"rows\").table()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## IO Operations" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Load from file" + "df_csv =op.load.csv(\"data/foo.csv\").table(5)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -2149,7 +1646,7 @@ "\n", "\n", "\n", - "
Viewing 5 of 19 rows / 8 columns
\n", + "
Viewing 4 of 4 rows / 5 columns
\n", "
1 partition(s)
\n", "\n", "\n", @@ -2157,8 +1654,8 @@ " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", + "\n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2316,35 +1745,33 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2352,35 +1779,33 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2388,35 +1813,33 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -2425,7 +1848,7 @@ "
\n", - "
id
\n", - "
1 (int)
\n", + "
Sepal length
\n", + "
1 (string)
\n", "
\n", " \n", " nullable\n", @@ -2167,8 +1664,8 @@ "
\n", - "
firstName
\n", - "
2 (string)
\n", + "
Sepal width
\n", + "
2 (double)
\n", "
\n", " \n", " nullable\n", @@ -2177,8 +1674,8 @@ "
\n", - "
lastName
\n", - "
3 (string)
\n", + "
Petal length
\n", + "
3 (double)
\n", "
\n", " \n", " nullable\n", @@ -2187,8 +1684,8 @@ "
\n", - "
billingId
\n", - "
4 (int)
\n", + "
Petal width
\n", + "
4 (string)
\n", "
\n", " \n", " nullable\n", @@ -2197,118 +1694,50 @@ "
\n", - "
product
\n", + "
Species
\n", "
5 (string)
\n", "
\n", " \n", " nullable\n", " \n", - "
\n", - " 		\n", - "
price
\n", - "
6 (int)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
birth
\n", - "
7 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - " 		\n", - "
dummyCol
\n", - "
8 (string)
\n", - "
\n", - " \n", - " nullable\n", - " \n", - "
\n", - "
\n", - "
1
\n", - " 		\n", - "
Luis
\n", - " 		\n", - "
Alvarez$$%!
\n", - " 		\n", - "
123
\n", - " 		\n", - "
Cake
\n", - " 		\n", - "
10
\n", - " 		\n", - "
1980/07/07
\n", - " 		\n", - "
never
\n", - "
\n", - "
2
\n", - " 		\n", - "
André
\n", - " 		\n", - "
Ampère
\n", - " 		\n", - "
423
\n", + "
5.0\n", + "
\n", " 		\n", - "
piza
\n", + "
3.6\n", + "
\n", " 		\n", - "
8
\n", + "
1.4\n", + "
\n", " 		\n", - "
1950/07/08
\n", + "
0.2\n", + "
\n", " 		\n", - "
gonna
\n", + "
I.⋅setosa\n", + "
\n", "
\n", - "
3
\n", - " 		\n", - "
NiELS
\n", - " 		\n", - "
Böhr//((%%
\n", - " 		\n", - "
551
\n", + "
5.0\n", + "
\n", " 		\n", - "
pizza
\n", + "
3.6\n", + "
\n", " 		\n", - "
8
\n", + "
1.4\n", + "
\n", " 		\n", - "
1990/07/09
\n", + "
0.2\n", + "
\n", " 		\n", - "
give
\n", + "
I.⋅setosa\n", + "
\n", "
\n", - "
4
\n", - " 		\n", - "
PAUL
\n", - " 		\n", - "
dirac$
\n", - " 		\n", - "
521
\n", + "
5.0\n", + "
\n", " 		\n", - "
pizza
\n", + "
3.6\n", + "
\n", " 		\n", - "
8
\n", + "
1.4\n", + "
\n", " 		\n", - "
1954/07/10
\n", + "
0.2\n", + "
\n", " 		\n", - "
you
\n", + "
I.⋅setosa\n", + "
\n", "
\n", - "
5
\n", - " 		\n", - "
Albert
\n", - " 		\n", - "
Einstein
\n", - " 		\n", - "
634
\n", + "
5.0\n", + "
\n", " 		\n", - "
pizza
\n", + "
3.6\n", + "
\n", " 		\n", - "
8
\n", + "
1.4\n", + "
\n", " 		\n", - "
1990/07/11
\n", + "
0.2\n", + "
\n", " 		\n", - "
up
\n", + "
I.⋅setosa\n", + "
\n", "
\n", "\n", "\n", - "
Viewing 5 of 19 rows / 8 columns
\n", + "
Viewing 4 of 4 rows / 5 columns
\n", "
1 partition(s)
\n" ], "text/plain": [ @@ -2437,12 +1860,12 @@ } ], "source": [ - "df_csv =op.load.csv(\"data/foo.csv\").table(5)" + "df_csv =op.load.tsv(\"data/foo.tsv\").table(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -2548,35 +1971,51 @@ " \n", " \n", " \n", - "
123
\n", + "
123\n", + "
\n", " \n", " \n", " \n", - "
1980/07/07
\n", + "
1980/07/07\n", + "
\n", " \n", " \n", " \n", - "
never
\n", + "
never\n", + "
\n", " \n", " \n", " \n", - "
Luis
\n", + "
Luis\n", + "
\n", " \n", " \n", " \n", - "
1
\n", + "
1\n", + "
\n", " \n", " \n", " \n", - "
Alvarez$$%!
\n", + "
Alvarez$$%!\n", + "
\n", " \n", " \n", " \n", - "
10
\n", + "
10\n", + "
\n", " \n", " \n", " \n", - "
Cake
\n", + "
Cake\n", + "
\n", " \n", " \n", " \n", @@ -2584,35 +2023,51 @@ " \n", " \n", " \n", - "
423
\n", + "
423\n", + "
\n", " \n", " \n", " \n", - "
1950/07/08
\n", + "
1950/07/08\n", + "
\n", " \n", " \n", " \n", - "
gonna
\n", + "
gonna\n", + "
\n", " \n", " \n", " \n", - "
André
\n", + "
André\n", + "
\n", " \n", " \n", " \n", - "
2
\n", + "
2\n", + "
\n", " \n", " \n", " \n", - "
Ampère
\n", + "
Ampère\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
piza
\n", + "
piza\n", + "
\n", " \n", " \n", " \n", @@ -2620,35 +2075,51 @@ " \n", " \n", " \n", - "
551
\n", + "
551\n", + "
\n", " \n", " \n", " \n", - "
1990/07/09
\n", + "
1990/07/09\n", + "
\n", " \n", " \n", " \n", - "
give
\n", + "
give\n", + "
\n", " \n", " \n", " \n", - "
NiELS
\n", + "
NiELS\n", + "
\n", " \n", " \n", " \n", - "
3
\n", + "
3\n", + "
\n", " \n", " \n", " \n", - "
Böhr//((%%
\n", + "
Böhr//((%%\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", @@ -2656,35 +2127,51 @@ " \n", " \n", " \n", - "
521
\n", + "
521\n", + "
\n", " \n", " \n", " \n", - "
1954/07/10
\n", + "
1954/07/10\n", + "
\n", " \n", " \n", " \n", - "
you
\n", + "
you\n", + "
\n", " \n", " \n", " \n", - "
PAUL
\n", + "
PAUL\n", + "
\n", " \n", " \n", " \n", - "
4
\n", + "
4\n", + "
\n", " \n", " \n", " \n", - "
dirac$
\n", + "
dirac$\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", @@ -2692,35 +2179,51 @@ " \n", " \n", " \n", - "
634
\n", + "
634\n", + "
\n", " \n", " \n", " \n", - "
1990/07/11
\n", + "
1990/07/11\n", + "
\n", " \n", " \n", " \n", - "
up
\n", + "
up\n", + "
\n", " \n", " \n", " \n", - "
Albert
\n", + "
Albert\n", + "
\n", " \n", " \n", " \n", - "
5
\n", + "
5\n", + "
\n", " \n", " \n", " \n", - "
Einstein
\n", + "
Einstein\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", @@ -2746,7 +2249,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -2852,35 +2355,51 @@ " \n", " \n", " \n", - "
1
\n", + "
1\n", + "
\n", " \n", " \n", " \n", - "
Luis
\n", + "
Luis\n", + "
\n", " \n", " \n", " \n", - "
Alvarez$$%!
\n", + "
Alvarez$$%!\n", + "
\n", " \n", " \n", " \n", - "
123
\n", + "
123\n", + "
\n", " \n", " \n", " \n", - "
Cake
\n", + "
Cake\n", + "
\n", " \n", " \n", " \n", - "
10
\n", + "
10\n", + "
\n", " \n", " \n", " \n", - "
1980/07/07
\n", + "
1980/07/07\n", + "
\n", " \n", " \n", " \n", - "
never
\n", + "
never\n", + "
\n", " \n", " \n", " \n", @@ -2888,35 +2407,51 @@ " \n", " \n", " \n", - "
2
\n", + "
2\n", + "
\n", " \n", " \n", " \n", - "
André
\n", + "
André\n", + "
\n", " \n", " \n", " \n", - "
Ampère
\n", + "
Ampère\n", + "
\n", " \n", " \n", " \n", - "
423
\n", + "
423\n", + "
\n", " \n", " \n", " \n", - "
piza
\n", + "
piza\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
1950/07/08
\n", + "
1950/07/08\n", + "
\n", " \n", " \n", " \n", - "
gonna
\n", + "
gonna\n", + "
\n", " \n", " \n", " \n", @@ -2924,35 +2459,51 @@ " \n", " \n", " \n", - "
3
\n", + "
3\n", + "
\n", " \n", " \n", " \n", - "
NiELS
\n", + "
NiELS\n", + "
\n", " \n", " \n", " \n", - "
Böhr//((%%
\n", + "
Böhr//((%%\n", + "
\n", " \n", " \n", " \n", - "
551
\n", + "
551\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
1990/07/09
\n", + "
1990/07/09\n", + "
\n", " \n", " \n", " \n", - "
give
\n", + "
give\n", + "
\n", " \n", " \n", " \n", @@ -2960,35 +2511,51 @@ " \n", " \n", " \n", - "
4
\n", + "
4\n", + "
\n", " \n", " \n", " \n", - "
PAUL
\n", + "
PAUL\n", + "
\n", " \n", " \n", " \n", - "
dirac$
\n", + "
dirac$\n", + "
\n", " \n", " \n", " \n", - "
521
\n", + "
521\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
1954/07/10
\n", + "
1954/07/10\n", + "
\n", " \n", " \n", " \n", - "
you
\n", + "
you\n", + "
\n", " \n", " \n", " \n", @@ -2996,35 +2563,51 @@ " \n", " \n", " \n", - "
5
\n", + "
5\n", + "
\n", " \n", " \n", " \n", - "
Albert
\n", + "
Albert\n", + "
\n", " \n", " \n", " \n", - "
Einstein
\n", + "
Einstein\n", + "
\n", " \n", " \n", " \n", - "
634
\n", + "
634\n", + "
\n", " \n", " \n", " \n", - "
pizza
\n", + "
pizza\n", + "
\n", " \n", " \n", " \n", - "
8
\n", + "
8\n", + "
\n", " \n", " \n", " \n", - "
1990/07/11
\n", + "
1990/07/11\n", + "
\n", " \n", " \n", " \n", - "
up
\n", + "
up\n", + "
\n", " \n", " \n", " \n", diff --git a/examples/new-api-sandbox.ipynb b/examples/new-api-sandbox.ipynb index 82cf173b9..25f289159 100644 --- a/examples/new-api-sandbox.ipynb +++ b/examples/new-api-sandbox.ipynb @@ -24,7 +24,19 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n", + " from collections import Callable\n", + "\n", + " You are using PySparkling of version 2.4.10, but your PySpark is of\n", + " version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n" + ] + } + ], "source": [ "from optimus import Optimus" ] @@ -41,10 +53,10 @@ "INFO:optimus:Just check that Spark and all necessary environments vars are present...\n", "INFO:optimus:-----\n", "INFO:optimus:SPARK_HOME=C:\\opt\\spark\\spark-2.3.1-bin-hadoop2.7\n", - "INFO:optimus:HADOOP_HOME=C:\\opt\\spark\\spark-2.3.1-bin-hadoop2.7\n", + "INFO:optimus:HADOOP_HOME=C:\\opt\\hadoop-2.7.7\n", "INFO:optimus:PYSPARK_PYTHON=C:\\Users\\argenisleon\\Anaconda3\\python.exe\n", "INFO:optimus:PYSPARK_DRIVER_PYTHON=jupyter\n", - "INFO:optimus:PYSPARK_SUBMIT_ARGS=--conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", + "INFO:optimus:PYSPARK_SUBMIT_ARGS=--packages com.databricks:spark-avro_2.11:4.0.0 --conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", "INFO:optimus:JAVA_HOME=C:\\java\n", "INFO:optimus:Pyarrow Installed\n", "INFO:optimus:-----\n", @@ -105,46 +117,925 @@ " border-bottom: 1px solid #cccccc;\n", " }\n", "\n", - " /* Profiler */\n", - " .main{\n", - " width:100%;\n", - " overflow:auto;\n", - " border-bottom:1px solid #eeeeee;\n", - " padding: 10px 0;\n", - " }\n", - " .panel_profiler{\n", - " margin-right:2%;\n", - " float:left;\n", - " padding-bottom:2%;\n", - " }\n", - " .panel_profiler tbody{\n", - " font-family:monospace;\n", - " }\n", - " .title_profiler{\n", - " padding:20px;\n", - " background-color: #eeeeee\n", - " }\n", - " .info{\n", - " overflow: auto\n", - " }\n", - " .main td, main th{\n", - " padding:0em\n", - " }\n", - " .panel_profiler td {\n", - " padding:0.2em\n", - " }\n", - " .none, .true{\n", - " color:#0000ff\n", - " }\n", - " .optimus_table th {\n", - " font-family:sans-serif;\n", - " }\n", + " /* Profiler */\n", + " .main{\n", + " width:100%;\n", + " overflow:auto;\n", + " border-bottom:1px solid #eeeeee;\n", + " padding: 10px 0;\n", + " }\n", + " .panel_profiler{\n", + " margin-right:2%;\n", + " float:left;\n", + " padding-bottom:2%;\n", + " }\n", + " .panel_profiler tbody{\n", + " font-family:monospace;\n", + " }\n", + " .title_profiler{\n", + " padding:20px;\n", + " background-color: #eeeeee\n", + " }\n", + " .info{\n", + " overflow: auto\n", + " }\n", + " .main td, main th{\n", + " padding:0em\n", + " }\n", + " .panel_profiler td {\n", + " padding:0.2em\n", + " }\n", + " .none, .true{\n", + " color:#0000ff\n", + " }\n", + " .optimus_table th {\n", + " font-family:sans-serif;\n", + " }\n", + "\n", + " .info_items{\n", + " font-family:sans-serif;\n", + " font-size:10px;\n", + " }\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "op= Optimus(master=\"local\", app_name= \"optimus\", verbose = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "ename": "AnalysisException", + "evalue": "'java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mPy4JJavaError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pyspark\\sql\\utils.py\u001b[0m in \u001b[0;36mdeco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m 62\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 64\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py4j\\protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m 327\u001b[0m \u001b[1;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 328\u001b[1;33m format(target_id, \".\", name), value)\n\u001b[0m\u001b[0;32m 329\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling o31.applySchemaToPythonRDD.\n: org.apache.spark.sql.AnalysisException: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:106)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\r\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\r\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\r\n\tat org.apache.spark.sql.SparkSession.internalCreateDataFrame(SparkSession.scala:577)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:752)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:737)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient\r\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.(HiveClientImpl.scala:114)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\r\n\t... 30 more\r\nCaused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient\r\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\r\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\r\n\t... 45 more\r\nCaused by: java.lang.reflect.InvocationTargetException\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\r\n\t... 51 more\r\nCaused by: javax.jdo.JDOFatalDataStoreException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\r\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\r\n\tat com.jolbox.bonecp.BoneCP.(BoneCP.java:416)\r\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\r\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\r\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\r\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\r\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\r\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\r\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\r\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\r\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199)\r\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\r\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.(HiveClientImpl.scala:114)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\r\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\r\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\r\n\tat org.apache.spark.sql.SparkSession.internalCreateDataFrame(SparkSession.scala:577)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:752)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:737)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\r\n\t... 113 more\r\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database C:\\Users\\argenisleon\\Documents\\Optimus\\examples\\metastore_db.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\r\n\t... 110 more\r\n------\r\n\nNestedThrowables:\njava.sql.SQLException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\r\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\r\n\tat com.jolbox.bonecp.BoneCP.(BoneCP.java:416)\r\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\r\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\r\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\r\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\r\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\r\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\r\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\r\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\r\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199)\r\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\r\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.(HiveClientImpl.scala:114)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\r\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\r\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\r\n\tat org.apache.spark.sql.SparkSession.internalCreateDataFrame(SparkSession.scala:577)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:752)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:737)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\r\n\t... 113 more\r\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database C:\\Users\\argenisleon\\Documents\\Optimus\\examples\\metastore_db.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\r\n\t... 110 more\r\n------\r\n\r\n\tat org.datanucleus.api.jdo.NucleusJDOHelper.getJDOExceptionForNucleusException(NucleusJDOHelper.java:436)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:788)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\r\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\r\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\r\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199)\r\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74)\r\n\t... 56 more\r\nCaused by: java.sql.SQLException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\r\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\r\n\tat com.jolbox.bonecp.BoneCP.(BoneCP.java:416)\r\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\r\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\r\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\r\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\r\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\r\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\r\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\r\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\r\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\r\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\r\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57)\r\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66)\r\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\r\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199)\r\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\r\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\r\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\r\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\r\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.(HiveClientImpl.scala:114)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\r\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\r\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\r\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\r\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\r\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\r\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\r\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\r\n\tat org.apache.spark.sql.SparkSession.internalCreateDataFrame(SparkSession.scala:577)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:752)\r\n\tat org.apache.spark.sql.SparkSession.applySchemaToPythonRDD(SparkSession.scala:737)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\r\n\t... 113 more\r\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database C:\\Users\\argenisleon\\Documents\\Optimus\\examples\\metastore_db.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\r\n\t... 110 more\r\n------\r\n\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat com.jolbox.bonecp.PoolUtil.generateSQLException(PoolUtil.java:192)\r\n\tat com.jolbox.bonecp.BoneCP.(BoneCP.java:422)\r\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\r\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\r\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\r\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\r\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\r\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\r\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\r\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\r\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\r\n\t... 85 more\r\nCaused by: java.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\r\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\r\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\r\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\r\n\tat com.jolbox.bonecp.BoneCP.(BoneCP.java:416)\r\n\t... 97 more\r\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4cbe88e2, see the next exception for details.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\r\n\t... 113 more\r\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database C:\\Users\\argenisleon\\Documents\\Optimus\\examples\\metastore_db.\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\r\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\r\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\r\n\tat java.security.AccessController.doPrivileged(Native Method)\r\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\r\n\t... 110 more\r\n", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[1;31mAnalysisException\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 43\u001b[0m [11.0], date(2011, 4, 10), datetime(2014, 6, 24), True, bytearray(\"15\", \"utf-8\"), None)\n\u001b[0;32m 44\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 45\u001b[1;33m ], infer_schema = True).h_repartition(1)\n\u001b[0m\u001b[0;32m 46\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Documents\\Optimus\\optimus\\functions.py\u001b[0m in \u001b[0;36mdata_frame\u001b[1;34m(cols, rows, infer_schema, pdf)\u001b[0m\n\u001b[0;32m 522\u001b[0m \u001b[0mstruct_fields\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mStructField\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mspecs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 523\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 524\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSpark\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minstance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mspark\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreateDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrows\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mStructType\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstruct_fields\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 525\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 526\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pyspark\\sql\\session.py\u001b[0m in \u001b[0;36mcreateDataFrame\u001b[1;34m(self, data, schema, samplingRatio, verifySchema)\u001b[0m\n\u001b[0;32m 691\u001b[0m \u001b[0mrdd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mschema\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_createFromLocal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprepare\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 692\u001b[0m \u001b[0mjrdd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSerDeUtil\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoJavaArray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrdd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_to_java_object_rdd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 693\u001b[1;33m \u001b[0mjdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jsparkSession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapplySchemaToPythonRDD\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjrdd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 694\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_wrapped\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 695\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mschema\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py4j\\java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m 1255\u001b[0m \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1256\u001b[0m return_value = get_return_value(\n\u001b[1;32m-> 1257\u001b[1;33m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[0;32m 1258\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1259\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pyspark\\sql\\utils.py\u001b[0m in \u001b[0;36mdeco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m 67\u001b[0m e.java_exception.getStackTrace()))\n\u001b[0;32m 68\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'org.apache.spark.sql.AnalysisException: '\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m': '\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 70\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'org.apache.spark.sql.catalyst.analysis'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 71\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m': '\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mAnalysisException\u001b[0m: 'java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;'" + ] + } + ], + "source": [ + "import pandas as pd\n", + "from pyspark.sql.types import *\n", + "from datetime import date, datetime\n", + "\n", + "df = op.create.df(\n", + " [\n", + " (\"names\", \"str\", True), \n", + " (\"height(ft)\",\"int\", True), \n", + " (\"function\", \"str\", True), \n", + " (\"rank\", \"int\", True), \n", + " (\"age\",\"int\",True),\n", + " (\"weight(t)\",\"float\",True),\n", + " (\"japanese name\", ArrayType(StringType()), True),\n", + " (\"last position seen\", \"str\", True),\n", + " (\"date arrival\", \"str\", True),\n", + " (\"last date seen\", \"str\", True),\n", + " (\"attributes\", ArrayType(FloatType()), True),\n", + " (\"DateType\"),\n", + " (\"Tiemstamp\"),\n", + " (\"Cybertronian\", \"bool\", True), \n", + " (\"function(binary)\",\"binary\", False),\n", + " (\"NullType\", \"null\", True),\n", + "\n", + " ],\n", + " [\n", + " (\"Optim'us\", 28, \"Leader\", 10, 5000000, 4.30, [\"Inochi\", \"Convoy\"], \"19.442735,-99.201111\", \"1980/04/10\",\n", + " \"2016/09/10\", [8.5344, 4300.0], date(2016, 9, 10), datetime(2014, 6, 24), True, bytearray(\"Leader\", \"utf-8\"),\n", + " None),\n", + " (\"bumbl#ebéé \", 17, \"Espionage\", 7, 5000000, 2.0, [\"Bumble\", \"Goldback\"], \"10.642707,-71.612534\", \"1980/04/10\",\n", + " \"2015/08/10\", [5.334, 2000.0], date(2015, 8, 10), datetime(2014, 6, 24), True, bytearray(\"Espionage\", \"utf-8\"),\n", + " None),\n", + " (\"ironhide&\", 26, \"Security\", 7, 5000000, 4.0, [\"Roadbuster\"], \"37.789563,-122.400356\", \"1980/04/10\",\n", + " \"2014/07/10\", [7.9248, 4000.0], date(2014, 6, 24), datetime(2014, 6, 24), True, bytearray(\"Security\", \"utf-8\"),\n", + " None),\n", + " (\"Jazz\", 13, \"First Lieutenant\", 8, 5000000, 1.80, [\"Meister\"], \"33.670666,-117.841553\", \"1980/04/10\",\n", + " \"2013/06/10\", [3.9624, 1800.0], date(2013, 6, 24), datetime(2014, 6, 24), True,\n", + " bytearray(\"First Lieutenant\", \"utf-8\"), None),\n", + " (\"Megatron\", None, \"None\", 10, 5000000, 5.70, [\"Megatron\"], None, \"1980/04/10\", \"2012/05/10\", [None, 5700.0],\n", + " date(2012, 5, 10), datetime(2014, 6, 24), True, bytearray(\"None\", \"utf-8\"), None),\n", + " (\"Metroplex_)^$\", 300, \"Battle Station\", 8, 5000000, None, [\"Metroflex\"], None, \"1980/04/10\", \"2011/04/10\",\n", + " [91.44, None], date(2011, 4, 10), datetime(2014, 6, 24), True, bytearray(\"Battle Station\", \"utf-8\"), None),\n", + " (\"1\", 2, \"3\", 4, 5, 6.0, [\"7\"], 8, \"1980/04/10\", \"2011/04/10\",\n", + " [11.0], date(2011, 4, 10), datetime(2014, 6, 24), True, bytearray(\"15\", \"utf-8\"), None)\n", + "\n", + " ], infer_schema = True).h_repartition(1)\n", + "df.table()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
Viewing 6 of 6 rows / 16 columns
\n", + "
1 partition(s)
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "
names
\n", + "
1 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
height(ft)
\n", + "
2 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
function
\n", + "
3 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
rank
\n", + "
4 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
age
\n", + "
5 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
weight(t)
\n", + "
6 (float)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
japanese name
\n", + "
7 (array<string>)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
last position seen
\n", + "
8 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
date arrival
\n", + "
9 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
last date seen
\n", + "
10 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
attributes
\n", + "
11 (array<float>)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
DateType
\n", + "
12 (date)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
Tiemstamp
\n", + "
13 (timestamp)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
Cybertronian
\n", + "
14 (boolean)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
function(binary)
\n", + "
15 (binary)
\n", + "
\n", + " \n", + " not nullable\n", + " \n", + "
\n", + " 		\n", + "
NullType
\n", + "
16 (null)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
Optim'us\n", + "
\n", + " 		\n", + "
28\n", + "
\n", + " 		\n", + "
Leader\n", + "
\n", + " 		\n", + "
10\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
4.300000190734863\n", + "
\n", + " 		\n", + "
['Inochi',⋅'Convoy']\n", + "
\n", + " 		\n", + "
19.442735,-99.201111\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2016/09/10\n", + "
\n", + " 		\n", + "
[8.53439998626709,⋅4300.0]\n", + "
\n", + " 		\n", + "
2016-09-10\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
bumbl#ebéé⋅⋅\n", + "
\n", + " 		\n", + "
17\n", + "
\n", + " 		\n", + "
Espionage\n", + "
\n", + " 		\n", + "
7\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
2.0\n", + "
\n", + " 		\n", + "
['Bumble',⋅'Goldback']\n", + "
\n", + " 		\n", + "
10.642707,-71.612534\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2015/08/10\n", + "
\n", + " 		\n", + "
[5.334000110626221,⋅2000.0]\n", + "
\n", + " 		\n", + "
2015-08-10\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
ironhide&\n", + "
\n", + " 		\n", + "
26\n", + "
\n", + " 		\n", + "
Security\n", + "
\n", + " 		\n", + "
7\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
4.0\n", + "
\n", + " 		\n", + "
['Roadbuster']\n", + "
\n", + " 		\n", + "
37.789563,-122.400356\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2014/07/10\n", + "
\n", + " 		\n", + "
[7.924799919128418,⋅4000.0]\n", + "
\n", + " 		\n", + "
2014-06-24\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
Jazz\n", + "
\n", + " 		\n", + "
13\n", + "
\n", + " 		\n", + "
First⋅Lieutenant\n", + "
\n", + " 		\n", + "
8\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
1.7999999523162842\n", + "
\n", + " 		\n", + "
['Meister']\n", + "
\n", + " 		\n", + "
33.670666,-117.841553\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2013/06/10\n", + "
\n", + " 		\n", + "
[3.962399959564209,⋅1800.0]\n", + "
\n", + " 		\n", + "
2013-06-24\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
Megatron\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
10\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
5.699999809265137\n", + "
\n", + " 		\n", + "
['Megatron']\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2012/05/10\n", + "
\n", + " 		\n", + "
[None,⋅5700.0]\n", + "
\n", + " 		\n", + "
2012-05-10\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
Metroplex_)^$\n", + "
\n", + " 		\n", + "
300\n", + "
\n", + " 		\n", + "
Battle⋅Station\n", + "
\n", + " 		\n", + "
8\n", + "
\n", + " 		\n", + "
5000000\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
['Metroflex']\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
1980/04/10\n", + "
\n", + " 		\n", + "
2011/04/10\n", + "
\n", + " 		\n", + "
[91.44000244140625,⋅None]\n", + "
\n", + " 		\n", + "
2011-04-10\n", + "
\n", + " 		\n", + "
2014-06-24⋅00:00:00\n", + "
\n", + " 		\n", + "
True\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "\n", "\n", - " .info_items{\n", - " font-family:sans-serif;\n", - " font-size:10px;\n", - " }\n", - "" + "
Viewing 6 of 6 rows / 16 columns
\n", + "
1 partition(s)
\n" ], "text/plain": [ "" @@ -155,12 +1046,12 @@ } ], "source": [ - "op= Optimus(verbose=True)" + "df.rows.select_by_dtypes(\"names\",\"str\").table()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -430,8 +1321,8 @@ " \n", " \n", " \n", - "
bytearray(b'Leader')\n", + "
None\n", "
\n", " \n", " \n", @@ -530,8 +1421,8 @@ " \n", " \n", " \n", - "
bytearray(b'Espionage')\n", + "
None\n", "
\n", " \n", " \n", @@ -630,8 +1521,8 @@ " \n", " \n", " \n", - "
bytearray(b'Security')\n", + "
None\n", "
\n", " \n", " \n", @@ -730,8 +1621,8 @@ " \n", " \n", " \n", - "
bytearray(b'First⋅Lieutenant')\n", + "
None\n", "
\n", " \n", " \n", @@ -830,8 +1721,8 @@ " \n", " \n", " \n", - "
bytearray(b'None')\n", + "
None\n", "
\n", " \n", " \n", @@ -930,8 +1821,8 @@ " \n", " \n", " \n", - "
bytearray(b'Battle⋅Station')\n", + "
None\n", "
\n", " \n", " \n", @@ -959,55 +1850,47 @@ } ], "source": [ - "import pandas as pd\n", - "from pyspark.sql.types import *\n", - "from datetime import date, datetime\n", - "\n", - "df = op.create.df(\n", - " [\n", - " (\"names\", \"str\", True), \n", - " (\"height(ft)\",\"int\", True), \n", - " (\"function\", \"str\", True), \n", - " (\"rank\", \"int\", True), \n", - " (\"age\",\"int\",True),\n", - " (\"weight(t)\",\"float\",True),\n", - " (\"japanese name\", ArrayType(StringType()), True),\n", - " (\"last position seen\", \"str\", True),\n", - " (\"date arrival\", \"str\", True),\n", - " (\"last date seen\", \"str\", True),\n", - " (\"attributes\", ArrayType(FloatType()), True),\n", - " (\"DateType\"),\n", - " (\"Tiemstamp\"),\n", - " (\"Cybertronian\", \"bool\", True), \n", - " (\"function(binary)\",\"binary\", False),\n", - " (\"NullType\", \"null\", True),\n", - "\n", - " ],\n", - " [\n", - " (\"Optim'us\", 28, \"Leader\", 10, 5000000, 4.30, [\"Inochi\", \"Convoy\"], \"19.442735,-99.201111\", \"1980/04/10\",\n", - " \"2016/09/10\", [8.5344, 4300.0], date(2016, 9, 10), datetime(2014, 6, 24), True, bytearray(\"Leader\", \"utf-8\"),\n", - " None),\n", - " (\"bumbl#ebéé \", 17, \"Espionage\", 7, 5000000, 2.0, [\"Bumble\", \"Goldback\"], \"10.642707,-71.612534\", \"1980/04/10\",\n", - " \"2015/08/10\", [5.334, 2000.0], date(2015, 8, 10), datetime(2014, 6, 24), True, bytearray(\"Espionage\", \"utf-8\"),\n", - " None),\n", - " (\"ironhide&\", 26, \"Security\", 7, 5000000, 4.0, [\"Roadbuster\"], \"37.789563,-122.400356\", \"1980/04/10\",\n", - " \"2014/07/10\", [7.9248, 4000.0], date(2014, 6, 24), datetime(2014, 6, 24), True, bytearray(\"Security\", \"utf-8\"),\n", - " None),\n", - " (\"Jazz\", 13, \"First Lieutenant\", 8, 5000000, 1.80, [\"Meister\"], \"33.670666,-117.841553\", \"1980/04/10\",\n", - " \"2013/06/10\", [3.9624, 1800.0], date(2013, 6, 24), datetime(2014, 6, 24), True,\n", - " bytearray(\"First Lieutenant\", \"utf-8\"), None),\n", - " (\"Megatron\", None, \"None\", 10, 5000000, 5.70, [\"Megatron\"], None, \"1980/04/10\", \"2012/05/10\", [None, 5700.0],\n", - " date(2012, 5, 10), datetime(2014, 6, 24), True, bytearray(\"None\", \"utf-8\"), None),\n", - " (\"Metroplex_)^$\", 300, \"Battle Station\", 8, 5000000, None, [\"Metroflex\"], None, \"1980/04/10\", \"2011/04/10\",\n", - " [91.44, None], date(2011, 4, 10), datetime(2014, 6, 24), True, bytearray(\"Battle Station\", \"utf-8\"), None),\n", - "\n", - " ], infer_schema = True).h_repartition(1)\n", - "df.table()" + "df.rows.drop_by_dtypes(\"names\",\"int\").table()" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'names': 'string',\n", + " 'height(ft)': 'int',\n", + " 'function': 'string',\n", + " 'rank': 'int',\n", + " 'age': 'int',\n", + " 'weight(t)': 'float',\n", + " 'japanese name': 'array',\n", + " 'last position seen': 'string',\n", + " 'date arrival': 'string',\n", + " 'last date seen': 'string',\n", + " 'attributes': 'array',\n", + " 'DateType': 'date',\n", + " 'Tiemstamp': 'timestamp',\n", + " 'Cybertronian': 'boolean',\n", + " 'function(binary)': 'binary',\n", + " 'NullType': 'null'}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.cols.dtypes()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1277,8 +2160,8 @@ " \n", " \n", " \n", - "
bytearray(b'Leader')\n", + "
None\n", "
\n", " \n", " \n", @@ -1377,8 +2260,8 @@ " \n", " \n", " \n", - "
bytearray(b'Espionage')\n", + "
None\n", "
\n", " \n", " \n", @@ -1477,8 +2360,8 @@ " \n", " \n", " \n", - "
bytearray(b'Security')\n", + "
None\n", "
\n", " \n", " \n", @@ -1577,8 +2460,8 @@ " \n", " \n", " \n", - "
bytearray(b'First⋅Lieutenant')\n", + "
None\n", "
\n", " \n", " \n", @@ -1677,8 +2560,8 @@ " \n", " \n", " \n", - "
bytearray(b'None')\n", + "
None\n", "
\n", " \n", " \n", @@ -1777,8 +2660,8 @@ " \n", " \n", " \n", - "
bytearray(b'Battle⋅Station')\n", + "
None\n", "
\n", " \n", " \n", @@ -12742,8 +13625,37 @@ } ], "source": [ - "!pip install altair\n", - "!pip install -U altair vega_datasets notebook vega" + "import pandas as pd\n", + "from pyspark.sql.types import *\n", + "from datetime import date, datetime\n", + "\n", + "df = op.create.df(\n", + " [\n", + " (\"names\", \"str\", True), \n", + " (\"height(ft)\",\"int\", True), \n", + " (\"function\", \"str\", True), \n", + " (\"rank\", \"int\", True), \n", + " ],\n", + " [\n", + " (\"Optim'us\", 28, \"Leader\", 10, 5000000, 4.30, [\"Inochi\", \"Convoy\"], \"19.442735,-99.201111\", \"1980/04/10\",\n", + " \"2016/09/10\", [8.5344, 4300.0], date(2016, 9, 10), datetime(2014, 6, 24), True, bytearray(\"Leader\", \"utf-8\"),\n", + " None),\n", + " (\"bumbl#ebéé \", 17, \"Espionage\", 7, 5000000, 2.0, [\"Bumble\", \"Goldback\"], \"10.642707,-71.612534\", \"1980/04/10\",\n", + " \"2015/08/10\", [5.334, 2000.0], date(2015, 8, 10), datetime(2014, 6, 24), True, bytearray(\"Espionage\", \"utf-8\"),\n", + " None),\n", + " (\"ironhide&\", 26, \"Security\", 7, 5000000, 4.0, [\"Roadbuster\"], \"37.789563,-122.400356\", \"1980/04/10\",\n", + " \"2014/07/10\", [7.9248, 4000.0], date(2014, 6, 24), datetime(2014, 6, 24), True, bytearray(\"Security\", \"utf-8\"),\n", + " None),\n", + " (\"Jazz\", 13, \"First Lieutenant\", 8, 5000000, 1.80, [\"Meister\"], \"33.670666,-117.841553\", \"1980/04/10\",\n", + " \"2013/06/10\", [3.9624, 1800.0], date(2013, 6, 24), datetime(2014, 6, 24), True,\n", + " bytearray(\"First Lieutenant\", \"utf-8\"), None),\n", + " (\"Megatron\", None, \"None\", 10, 5000000, 5.70, [\"Megatron\"], None, \"1980/04/10\", \"2012/05/10\", [None, 5700.0],\n", + " date(2012, 5, 10), datetime(2014, 6, 24), True, bytearray(\"None\", \"utf-8\"), None),\n", + " (\"Metroplex_)^$\", 300, \"Battle Station\", 8, 5000000, None, [\"Metroflex\"], None, \"1980/04/10\", \"2011/04/10\",\n", + " [91.44, None], date(2011, 4, 10), datetime(2014, 6, 24), True, bytearray(\"Battle Station\", \"utf-8\"), None),\n", + "\n", + " ], infer_schema = True).h_repartition(1)\n", + "df.table()" ] }, { diff --git a/images/box.png b/images/box.png index b0b919d09..9794681ae 100644 Binary files a/images/box.png and b/images/box.png differ diff --git a/images/frequency.png b/images/frequency.png index 5ca6595d7..704f7c31f 100644 Binary files a/images/frequency.png and b/images/frequency.png differ diff --git a/images/hist.png b/images/hist.png index 306604922..a749f9d06 100644 Binary files a/images/hist.png and b/images/hist.png differ diff --git a/images/profiler.png b/images/profiler.png index ed5088606..b4c283d9c 100644 Binary files a/images/profiler.png and b/images/profiler.png differ diff --git a/images/profiler1.png b/images/profiler1.png index d199f3807..9b25cc170 100644 Binary files a/images/profiler1.png and b/images/profiler1.png differ diff --git a/images/profiler_numeric.png b/images/profiler_numeric.png new file mode 100644 index 000000000..a69fb054f Binary files /dev/null and b/images/profiler_numeric.png differ diff --git a/images/scatter.png b/images/scatter.png index 755ce48b6..d85cf9137 100644 Binary files a/images/scatter.png and b/images/scatter.png differ diff --git a/images/table.png b/images/table.png index 872fed015..b501d550b 100644 Binary files a/images/table.png and b/images/table.png differ diff --git a/images/table1.png b/images/table1.png index 872fed015..b501d550b 100644 Binary files a/images/table1.png and b/images/table1.png differ diff --git a/images/table2.png b/images/table2.png index 229b3b5a5..8e3259b52 100644 Binary files a/images/table2.png and b/images/table2.png differ diff --git a/images/table3.png b/images/table3.png index afd8d89ad..d3d955d22 100644 Binary files a/images/table3.png and b/images/table3.png differ diff --git a/images/table4.png b/images/table4.png index 10fd01dbd..bd5fb9440 100644 Binary files a/images/table4.png and b/images/table4.png differ diff --git a/images/table5.png b/images/table5.png index 1b6db93fc..3f13db92c 100644 Binary files a/images/table5.png and b/images/table5.png differ diff --git a/images/table6.png b/images/table6.png index 55f392bb4..aed92ed8b 100644 Binary files a/images/table6.png and b/images/table6.png differ diff --git a/optimus/dataframe/columns.py b/optimus/dataframe/columns.py index 2c3543f3d..746c065bc 100644 --- a/optimus/dataframe/columns.py +++ b/optimus/dataframe/columns.py @@ -37,6 +37,8 @@ from optimus.profiler.functions import bucketizer from optimus.profiler.functions import create_buckets +from optimus.functions import append as _append + def cols(self): @add_attr(cols) @@ -83,7 +85,7 @@ def append(cols_values=None): if is_list_of_dataframes(cols_values): dfs = cols_values dfs.insert(0, self) - df_result = append(dfs, like="columns") + df_result = _append(dfs, like="columns") elif is_list_of_tuples(cols_values): df_result = self @@ -1670,12 +1672,15 @@ def dtypes(columns="*"): return format_dict({col_name: data_types[col_name] for col_name in columns}) @add_attr(cols) - def names(filter_by_column_dtypes=None): + def names(col_names="*", filter_by_column_dtypes=None, invert=False): """ - Get column names + Get columns names + :param col_names: Columns names to be processed '*' for all or a list of column names + :param filter_by_column_dtypes: Data type used to select the columns + :param invert: Invert the columns selection :return: """ - columns = parse_columns(self, "*", filter_by_column_dtypes=filter_by_column_dtypes) + columns = parse_columns(self, col_names, filter_by_column_dtypes=filter_by_column_dtypes, invert=invert) return columns @add_attr(cols) diff --git a/optimus/dataframe/extension.py b/optimus/dataframe/extension.py index 0518a1738..43561e32c 100644 --- a/optimus/dataframe/extension.py +++ b/optimus/dataframe/extension.py @@ -1,3 +1,4 @@ +import json import os import humanize @@ -12,11 +13,11 @@ from pyspark.sql import functions as F from pyspark.sql.types import * -from optimus import RaiseIt +from optimus.helpers.raiseit import RaiseIt from optimus.helpers.checkit import is_str, is_column_a from optimus.helpers.convert import val_to_list, one_list_to_val from optimus.helpers.decorators import * -from optimus.helpers.functions import parse_columns, collect_as_dict, random_int, traverse, print_html +from optimus.helpers.functions import parse_columns, collect_as_dict, random_int, traverse, print_html, json_converter from optimus.helpers.logger import logger from optimus.profiler.templates.html import HEADER, FOOTER from optimus.spark import Spark @@ -38,7 +39,7 @@ def to_json(self): :param self: :return: """ - return collect_as_dict(self.collect()) + return json.loads(json.dumps(collect_as_dict(self), ensure_ascii=False, default=json_converter)) @add_method(DataFrame) diff --git a/optimus/helpers/functions.py b/optimus/helpers/functions.py index 4a3f8284e..3528d91b1 100644 --- a/optimus/helpers/functions.py +++ b/optimus/helpers/functions.py @@ -1,3 +1,4 @@ +import collections import datetime import inspect import json @@ -30,7 +31,6 @@ def infer(value): :return: Spark data type """ result = None - # print(v) if value is None: result = "null" @@ -120,10 +120,10 @@ def collect_as_list(df): return df.rdd.flatMap(lambda x: x).collect() -def collect_as_dict(value): +def collect_as_dict(df): """ Return a dict from a Collect result - :param value: + :param df: :return: """ @@ -133,8 +133,15 @@ def collect_as_dict(value): if len(dict_result) == 1: dict_result = next(iter(dict_result.values())) else: - dict_result = [v.asDict() for v in value] - + col_names = parse_columns(df, "*") + + # Because asDict can return messed columns names we order + for row in df.collect(): + _row = row.asDict() + r = collections.OrderedDict() + for col in col_names: + r[col] = _row[col] + dict_result.append(r) return dict_result @@ -282,7 +289,6 @@ def escape_columns(columns): escaped_columns = "`" + columns + "`" else: escaped_columns.append(columns) - # print(escaped_columns) return escaped_columns @@ -306,7 +312,7 @@ def get_output_cols(input_cols, output_cols): def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column_dtypes=None, - accepts_missing_cols=False): + accepts_missing_cols=False, invert=False): """ Return a list of columns and check that columns exists in the dataframe Accept '*' as parameter in which case return a list of all columns in the dataframe. @@ -319,14 +325,13 @@ def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column :param is_regex: Use True is col_attrs is a regex :param filter_by_column_dtypes: A data type for which a columns list is going be filtered :param accepts_missing_cols: if true not check if column exist in the dataframe + :param invert: Invert the final selection. For example if you want to select not integers + :return: A list of columns string names """ attrs = None - # ensure that cols_args is a list - # cols_args = val_to_list(cols_args) - # if columns value is * get all dataframes columns if is_regex is True: r = re.compile(cols_args[0]) @@ -335,10 +340,6 @@ def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column elif cols_args == "*" or cols_args is None: cols = df.columns - - # Return filtered columns - # columns_filtered = list(set(columns) - set(columns_filtered)) - # In case we have a list of tuples we use the first element of the tuple is taken as the column name # and the rest as params. We can use the param in a custom function as follow # def func(attrs): attrs return (1,2) and (3,4) @@ -362,13 +363,15 @@ def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column check_for_missing_columns(df, cols) # Filter by column data type - filter_by_column_dtypes = val_to_list(filter_by_column_dtypes) + if filter_by_column_dtypes is not None: + filter_by_column_dtypes = val_to_list(filter_by_column_dtypes) columns_residual = None # If necessary filter the columns by data type - if is_list_of_strings(filter_by_column_dtypes): + if filter_by_column_dtypes: # Get columns for every data type + columns_filtered = filter_col_name_by_dtypes(df, filter_by_column_dtypes) # Intersect the columns filtered per data type from the whole dataframe with the columns passed to the function @@ -378,10 +381,13 @@ def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column columns_residual = list(OrderedSet(cols) - OrderedSet(columns_filtered)) else: final_columns = cols - # final_columns = escape_columns(final_columns) + # Return cols or cols an params cols_params = [] + if invert: + final_columns = list(OrderedSet(cols) - OrderedSet(final_columns)) + if get_args is True: cols_params = final_columns, attrs elif get_args is False: @@ -389,9 +395,9 @@ def parse_columns(df, cols_args, get_args=False, is_regex=None, filter_by_column else: RaiseIt.value_error(get_args, ["True", "False"]) - if columns_residual: - print(",".join(escape_columns(columns_residual)), "column(s) was not processed because is/are not", - ",".join(filter_by_column_dtypes)) + # if columns_residual: + # print(",".join(escape_columns(columns_residual)), "column(s) was not processed because is/are not", + # ",".join(filter_by_column_dtypes)) return cols_params @@ -533,7 +539,7 @@ def json_enconding(obj): :param obj: :return: """ - return json.dumps(obj, sort_keys=True, default=json_converter) + return json.dumps(obj, default=json_converter) def debug(value): diff --git a/optimus/io/jdbc.py b/optimus/io/jdbc.py index 91ba83aa1..dea9e0a45 100644 --- a/optimus/io/jdbc.py +++ b/optimus/io/jdbc.py @@ -130,7 +130,7 @@ def table_to_df(self, table_name, columns="*", limit=None): def execute(self, query, limit=None): """ Execute a SQL query - :param limit: default limit the whole query. We play defensive here in case the result is a big chunck of data + :param limit: default limit the whole query. We play defensive here in case the result is a big chunk of data :param query: SQL query string :return: """ @@ -146,6 +146,23 @@ def execute(self, query, limit=None): .option("password", self.password) \ .load() + def df_to_table(self, df, table, mode="overwrite"): + """ + Send a dataframe to the database + :param df: + :param table: + :param mode + :return: + """ + return df.write \ + .format("jdbc") \ + .mode(mode) \ + .option("url", self.url) \ + .option("dbtable", table) \ + .option("user", self.user) \ + .option("password", self.password) \ + .save() + @staticmethod def _limit(limit=None): """ @@ -167,7 +184,7 @@ class Table: def __init__(self, db): self.db = db - def show(self, table_names="*", limit="all"): + def show(self, table_names="*", limit=None): db = self.db if table_names is "*": diff --git a/optimus/io/load.py b/optimus/io/load.py index 181de57ae..c0b4a3147 100644 --- a/optimus/io/load.py +++ b/optimus/io/load.py @@ -59,10 +59,23 @@ def json(path, multiline=False, *args, **kwargs): raise return replace_columns_special_characters(df) + @staticmethod + def tsv(path, header='true', infer_schema='true', *args, **kwargs): + """ + Return a dataframe from a tsv file. + :param path: path or location of the file. + :param header: tell the function whether dataset has a header row. 'true' default. + :param infer_schema: infers the input schema automatically from data. + It requires one extra pass over the data. 'true' default. + + :return: + """ + return Load.csv(path, sep='\t', header=header, infer_schema=infer_schema, *args, **kwargs) + @staticmethod def csv(path, sep=',', header='true', infer_schema='true', *args, **kwargs): """ - Return a dataframe from a csv file.. It is the same read.csv Spark function with some predefined + Return a dataframe from a csv file. It is the same read.csv Spark function with some predefined params :param path: path or location of the file. diff --git a/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar b/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar new file mode 100644 index 000000000..007c39299 Binary files /dev/null and b/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar differ diff --git a/optimus/jars/mysql-connector-java-8.0.16.jar b/optimus/jars/mysql-connector-java-8.0.16.jar new file mode 100644 index 000000000..acd5ea69a Binary files /dev/null and b/optimus/jars/mysql-connector-java-8.0.16.jar differ diff --git a/optimus/jars/ojdbc7.jar b/optimus/jars/ojdbc7.jar new file mode 100644 index 000000000..fd38a6c4a Binary files /dev/null and b/optimus/jars/ojdbc7.jar differ diff --git a/optimus/jars/postgresql-42.2.5.jar b/optimus/jars/postgresql-42.2.5.jar new file mode 100644 index 000000000..d89d4331a Binary files /dev/null and b/optimus/jars/postgresql-42.2.5.jar differ diff --git a/optimus/ml/feature.py b/optimus/ml/feature.py index 919dfb527..58e5ab858 100644 --- a/optimus/ml/feature.py +++ b/optimus/ml/feature.py @@ -1,10 +1,9 @@ from pyspark.ml import feature, Pipeline from pyspark.ml.feature import StringIndexer, IndexToString, OneHotEncoder, VectorAssembler, Normalizer -from pyspark.ml.linalg import DenseVector, VectorUDT -from pyspark.sql import functions as F -from optimus.helpers.checkit import is_dataframe, is_ +from optimus.helpers.checkit import is_dataframe, is_, is_str from optimus.helpers.functions import parse_columns +from optimus.helpers.raiseit import RaiseIt def n_gram(df, input_col, n=2): @@ -29,7 +28,7 @@ def n_gram(df, input_col, n=2): return df_model, tfidf_model -def string_to_index(df, input_cols): +def string_to_index(df, input_cols, **kargs): """ Maps a string column of labels to an ML column of label indices. If the input column is numeric, we cast it to string and index the string values. @@ -40,7 +39,7 @@ def string_to_index(df, input_cols): input_cols = parse_columns(df, input_cols) - indexers = [StringIndexer(inputCol=column, outputCol=column + "_index").fit(df) for column in + indexers = [StringIndexer(inputCol=column, outputCol=column + "_index", **kargs).fit(df) for column in list(set(input_cols))] pipeline = Pipeline(stages=indexers) @@ -49,7 +48,7 @@ def string_to_index(df, input_cols): return df -def index_to_string(df, input_cols): +def index_to_string(df, input_cols, **kargs): """ Maps a column of indices back to a new column of corresponding string values. The index-string mapping is either from the ML attributes of the input column, or from user-supplied labels (which take precedence over @@ -61,7 +60,7 @@ def index_to_string(df, input_cols): input_cols = parse_columns(df, input_cols) - indexers = [IndexToString(inputCol=column, outputCol=column + "_string") for column in + indexers = [IndexToString(inputCol=column, outputCol=column + "_string", **kargs) for column in list(set(input_cols))] pipeline = Pipeline(stages=indexers) @@ -70,7 +69,7 @@ def index_to_string(df, input_cols): return df -def one_hot_encoder(df, input_cols): +def one_hot_encoder(df, input_cols, **kargs): """ Maps a column of label indices to a column of binary vectors, with at most a single one-value. :param df: Dataframe to be transformed @@ -80,7 +79,7 @@ def one_hot_encoder(df, input_cols): input_cols = parse_columns(df, input_cols) - encode = [OneHotEncoder(inputCol=column, outputCol=column + "_encoded") for column in + encode = [OneHotEncoder(inputCol=column, outputCol=column + "_encoded", **kargs) for column in list(set(input_cols))] pipeline = Pipeline(stages=encode) @@ -119,31 +118,16 @@ def normalizer(df, input_cols, p=2.0): """ # Check if columns argument must be a string or list datatype: + if is_(input_cols, [str, list]): + RaiseIt.type_error(input_cols, [str, list]) - assert isinstance(input_cols, (str, list)), \ - "Error: %s argument must be a string or a list." % "input_cols" - - if isinstance(input_cols, str): + if is_str(input_cols): input_cols = [input_cols] - assert isinstance(p, (float, int)), "Error: p argument must be a numeric value." - - # Convert ArrayType() column to DenseVector - def arr_to_vec(arr_column): - """ - :param arr_column: Column name - :return: Returns DenseVector by converting an ArrayType() column - """ - return DenseVector(arr_column) - - # User-Defined function - # TODO: use apply() to use Pyarrow - udf_arr_to_vec = F.udf(arr_to_vec, VectorUDT()) + if is_(input_cols, [float, int]): + RaiseIt.type_error(input_cols, [float, int]) - # Check for columns which are not DenseVector types and convert them into DenseVector - for col in input_cols: - if not is_(df[col], DenseVector): - df = df.withColumn(col, udf_arr_to_vec(df[col])) + df = df.cols.cast(input_cols, "vector") normal = [Normalizer(inputCol=column, outputCol=column + "_normalized", p=p) for column in list(set(input_cols))] diff --git a/optimus/ml/models.py b/optimus/ml/models.py index 774ba1d05..a5ccac2a7 100644 --- a/optimus/ml/models.py +++ b/optimus/ml/models.py @@ -1,19 +1,13 @@ from pyspark.ml import feature, classification from pyspark.ml.classification import RandomForestClassifier, DecisionTreeClassifier, GBTClassifier from pyspark.sql.functions import * -from pyspark.sql.session import SparkSession from pysparkling import * from pysparkling.ml import H2OAutoML, H2ODeepLearning, H2OXGBoost, H2OGBM -from optimus.helpers.checkit import is_dataframe +from optimus.helpers.checkit import is_dataframe, is_str from optimus.helpers.functions import parse_columns from optimus.ml.feature import string_to_index, vector_assembler - -# from optimus.spark import Spark -# spark = Spark.instance.spark() -# TODO: not use getOrCreate use the singleton in optimus.spark - -spark = SparkSession.builder.getOrCreate() +from optimus.spark import Spark class ML: @@ -77,7 +71,8 @@ def decision_tree(df, columns, input_col, **kargs): columns = parse_columns(df, columns) - assert isinstance(input_col, str), "Error, input column must be a string" + if not is_str(input_col): + raise TypeError("Error, input column must be a string") data = df.select(columns) feats = data.columns @@ -109,7 +104,8 @@ def gbt(df, columns, input_col, **kargs): columns = parse_columns(df, columns) - assert isinstance(input_col, str), "Error, input column must be a string" + if not is_str(input_col): + raise TypeError("Error, input column must be a string") data = df.select(columns) feats = data.columns @@ -129,7 +125,7 @@ def gbt(df, columns, input_col, **kargs): @staticmethod def h2o_automl(df, label, columns, **kargs): - H2OContext.getOrCreate(spark) + H2OContext.getOrCreate(Spark.instance.spark) df_sti = string_to_index(df, input_cols=label) df_va = vector_assembler(df_sti, input_cols=columns) @@ -137,7 +133,7 @@ def h2o_automl(df, label, columns, **kargs): maxRuntimeSecs=60, # 1 minutes seed=1, maxModels=3, - predictionCol=label + "_index", + labelCol=label + "_index", **kargs) model = automl.fit(df_va) @@ -150,7 +146,7 @@ def h2o_automl(df, label, columns, **kargs): @staticmethod def h2o_deeplearning(df, label, columns, **kargs): - H2OContext.getOrCreate(spark) + H2OContext.getOrCreate(Spark.instance.spark) df_sti = string_to_index(df, input_cols=label) df_va = vector_assembler(df_sti, input_cols=columns) @@ -160,7 +156,7 @@ def h2o_deeplearning(df, label, columns, **kargs): l2=0.0, hidden=[200, 200], featuresCols=columns, - predictionCol=label, + labelCol=label, **kargs) model = h2o_deeplearning.fit(df_va) df_raw = model.transform(df_va) @@ -172,13 +168,13 @@ def h2o_deeplearning(df, label, columns, **kargs): @staticmethod def h2o_xgboost(df, label, columns, **kargs): - H2OContext.getOrCreate(spark) + H2OContext.getOrCreate(Spark.instance.spark) df_sti = string_to_index(df, input_cols=label) df_va = vector_assembler(df_sti, input_cols=columns) h2o_xgboost = H2OXGBoost(convertUnknownCategoricalLevelsToNa=True, featuresCols=columns, - predictionCol=label, + labelCol=label, **kargs) model = h2o_xgboost.fit(df_va) df_raw = model.transform(df_va) @@ -190,14 +186,14 @@ def h2o_xgboost(df, label, columns, **kargs): @staticmethod def h2o_gbm(df, label, columns, **kargs): - H2OContext.getOrCreate(spark) + H2OContext.getOrCreate(Spark.instance.spark) df_sti = string_to_index(df, input_cols=label) df_va = vector_assembler(df_sti, input_cols=columns) h2o_gbm = H2OGBM(ratio=0.8, seed=1, featuresCols=columns, - predictionCol=label, + labelCol=label, **kargs) model = h2o_gbm.fit(df_va) df_raw = model.transform(df_va) diff --git a/optimus/optimus.py b/optimus/optimus.py index 34f471a90..98b87f12b 100644 --- a/optimus/optimus.py +++ b/optimus/optimus.py @@ -1,12 +1,14 @@ import os import sys +from pathlib import Path from shutil import rmtree -from deepdiff import DeepDiff # For Deep Difference of 2 objects +from deepdiff import DeepDiff from pyspark.sql import DataFrame from optimus.enricher import Enricher from optimus.functions import append, Create +from optimus.helpers.checkit import is_list from optimus.helpers.constants import * from optimus.helpers.convert import val_to_list from optimus.helpers.functions import print_html, print_json @@ -31,6 +33,7 @@ def __init__(self, session=None, master="local[*]", app_name="optimus", checkpoi repositories=None, packages=None, jars=None, + driver_class_path=None, options=None, additional_options=None, queue_url=None, @@ -84,10 +87,15 @@ def __init__(self, session=None, master="local[*]", app_name="optimus", checkpoi self.packages = packages self.repositories = repositories - if jars is None: - jars = {} - + # Jars self.jars = jars + self._add_jars(jars) + + # Class Drive Path + self.driver_class_path = driver_class_path + self._add_driver_class_path(driver_class_path) + + # Additional Options self.additional_options = additional_options self.verbose(verbose) @@ -96,7 +104,20 @@ def __init__(self, session=None, master="local[*]", app_name="optimus", checkpoi # TODO: # if the Spark 2.4 version is going to be used this is not neccesesary. # Maybe we can check a priori which version fo Spark is going to be used - # self._add_spark_packages(["com.databricks:spark-avro_2.11:4.0.0"]) + self._add_spark_packages(["com.databricks:spark-avro_2.11:4.0.0"]) + + def c(files): + return [Path(path + file).as_posix() for file in files] + + path = os.path.dirname(os.path.abspath(__file__)) + + # Add databases jars + self._add_jars(["../jars/RedshiftJDBC42-1.2.16.1027.jar", "../jars/mysql-connector-java-8.0.16.jar", + "../jars/ojdbc7.jar", "../jars/postgresql-42.2.5.jar"]) + + self._add_driver_class_path( + c(["//jars//RedshiftJDBC42-1.2.16.1027.jar", "//jars//mysql-connector-java-8.0.16.jar", + "//jars//ojdbc7.jar", "//jars//postgresql-42.2.5.jar"])) self._start_session() @@ -299,20 +320,22 @@ def append(dfs, like): """ return append(dfs, like) + def _setup_repositories(self): + if self.repositories: + return '--repositories {}'.format(','.join(self.repositories)) + else: + return '' + + # Spark Package def _add_spark_packages(self, packages): """ Define the Spark packages that must be loaded at start time :param packages: :return: """ - for p in packages: - self.packages.append(p) - def _setup_repositories(self): - if self.repositories: - return '--repositories {}'.format(','.join(self.repositories)) - else: - return '' + for p in val_to_list(packages): + self.packages.append(p) def _setup_packages(self): if self.packages: @@ -320,12 +343,37 @@ def _setup_packages(self): else: return '' + # Jar + def _add_jars(self, jar): + if self.jars is None: + self.jars = [] + + if is_list(jar): + for j in val_to_list(jar): + self.jars.append(j) + def _setup_jars(self): if self.jars: return '--jars {}'.format(','.join(self.jars)) else: return '' + # Driver class path + def _add_driver_class_path(self, driver_class_path): + if self.driver_class_path is None: + self.driver_class_path = [] + + if is_list(driver_class_path): + for d in val_to_list(driver_class_path): + self.driver_class_path.append(d) + + def _setup_driver_class_path(self): + if self.driver_class_path: + return '--driver-class-path {}'.format(';'.join(self.driver_class_path)) + else: + return '' + + # Options def _setup_options(self, additional_options): options = {} @@ -359,6 +407,7 @@ def _start_session(self): self._setup_repositories(), self._setup_packages(), self._setup_jars(), + self._setup_driver_class_path(), self._setup_options(self.additional_options), 'pyspark-shell', ] diff --git a/optimus/profiler/functions.py b/optimus/profiler/functions.py index 4885098a3..ab46db83d 100644 --- a/optimus/profiler/functions.py +++ b/optimus/profiler/functions.py @@ -42,7 +42,7 @@ def write_json(data, path): """ try: with open(path, 'w', encoding='utf-8') as outfile: - json.dump(data, outfile, sort_keys=True, indent=4, ensure_ascii=False, default=json_converter) + json.dump(data, outfile, indent=4, ensure_ascii=False, default=json_converter) except IOError: pass diff --git a/optimus/spark.py b/optimus/spark.py index 0f3f3e350..6be46bc06 100644 --- a/optimus/spark.py +++ b/optimus/spark.py @@ -31,17 +31,12 @@ def create(self, master="local[*]", app_name="optimus"): logger.print("-----") logger.print(STARTING_SPARK) - # print(os.environ['PYSPARK_SUBMIT_ARGS']) - # Build the spark session self._spark = SparkSession.builder \ .appName(app_name) \ .master(master) \ - .config("spark.executor.heartbeatInterval", "110") \ - .config("spark.jars.packages", "ml.combust.mleap:mleap-spark_2.11:0.13.0") \ .getOrCreate() - # .option("driver", "org.postgresql.Driver") self._sc = self._spark.sparkContext logger.print("Spark Version:" + self._sc.version) diff --git a/optimus/version.py b/optimus/version.py index a40d50a0e..342c856eb 100644 --- a/optimus/version.py +++ b/optimus/version.py @@ -5,5 +5,5 @@ def _safe_int(string): return string -__version__ = '2.2.51' +__version__ = '2.2.6' VERSION = tuple(_safe_int(x) for x in __version__.split('.')) diff --git a/readme_.ipynb b/readme_.ipynb index 957b34fc1..c305f71ee 100644 --- a/readme_.ipynb +++ b/readme_.ipynb @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -191,7 +191,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -302,7 +302,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -336,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -390,7 +390,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "lines_to_next_cell": 2 }, @@ -400,7 +400,14 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -435,17 +442,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "`japanese name`,`attributes`,`tiemstamp`,`nulltype` column(s) was not processed because is/are not byte,short,big,int,double,float,string,date,bool\n" - ] - } - ], + "outputs": [], "source": [ "# This is a custom function\n", "def func(value, arg):\n", @@ -480,7 +479,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -488,7 +487,14 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -506,120 +512,1454 @@ } ], "source": [ - "df.table_image(\"images/table1.png\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Into this" + "df.table_image(\"images/table1.png\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Into this" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", + "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", + "Done \n" + ] + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "new_df.table_image(\"images/table2.png\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 0 + }, + "source": [ + "Note that you can use Optimus functions and Spark functions(`.WithColumn()`) and all the df function availables in a Spark Dataframe at the same time. To know about all the Optimus functionality please go to this [notebooks](examples/)\n", + "\n", + "### Custom functions\n", + "Spark have multiple ways to transform your data like rdd, Column Expression ,udf and pandas udf. In Optimus we create the `apply()` and `apply_expr` which handle all the implementation complexity.\n", + "\n", + "Here you apply a function to the \"billingid\" column. Sum 1 and 2 to the current column value. All powered by Pandas UDF" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", + "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", + "Done \n" + ] + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def func(value, args):\n", + " return value + args[0] + args[1]\n", + "\n", + "df.cols.apply(\"height(ft)\",func,\"int\", [1,2]).table_image(\"images/table3.png\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to apply a Column Expression use `apply_expr()` like this. In this case we pasa an argument 10 to divide the actual column value" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", + "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", + "Done \n" + ] + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pyspark.sql import functions as F\n", + "\n", + "def func(col_name, args):\n", + " return F.col(col_name)/20\n", + "\n", + "df.cols.apply(\"height(ft)\", func=func, args=20).table_image(\"images/table4.png\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can change the table output back to ascii if you which" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "op.output(\"ascii\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To return to HTML just:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "op.output(\"html\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data profiling\n", + "\n", + "Optimus comes with a powerful and unique data profiler. Besides basic and advance stats like min, max, kurtosis, mad etc, \n", + "it also let you know what type of data has every column. For example if a string column have string, integer, float, bool, date Optimus can give you an unique overview about your data. \n", + "Just run `df.profile(\"*\")` to profile all the columns. For more info about the profiler please go to this [notebook](examples/new-api-profiler.ipynb).\n", + "\n", + "Let's load a \"big\" dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "df = op.load.csv(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/Meteorite_Landings.csv\").h_repartition()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numeric" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Including 'nan' as Null in processing 'name'\n", + "Including 'nan' as Null in processing 'nametype'\n", + "Including 'nan' as Null in processing 'recclass'\n", + "Including 'nan' as Null in processing 'fall'\n", + "Including 'nan' as Null in processing 'year'\n", + "Including 'nan' as Null in processing 'GeoLocation'\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "

Overview

\n", + "
\n", + "
\n", + "
\n", + "

Dataset info

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + "
Number of columns10
Number of rows45716
Total Missing (%)0.49%
Total size in memory-1 Bytes
\n", + "
\n", + "
\n", + "

Column types

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + "
String0
Numeric1
Date0
Bool0
Array0
Not available0
\n", + "
\n", + "
\n", + "\n", + "\n", + "
\n", + "
\n", + "\n", + " \n", + "\n", + "
\n", + "
\n", + "

mass (g)

\n", + " numeric\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unique 12497
Unique (%) 27.336
Missing0.0
Missing (%)0
\n", + "
\n", + "

\n", + " Datatypes\n", + "

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + "
\n", + " String\n", + " \n", + " 0\n", + "
\n", + " Integer\n", + " \n", + " 0\n", + "
\n", + " Float\n", + " \n", + " 0\n", + "
\n", + " Bool\n", + " \n", + " 0\n", + "
\n", + " Date\n", + " \n", + " 0\n", + "
\n", + " Missing\n", + " \n", + " 0\n", + "
\n", + " Null\n", + " \n", + " 131\n", + "
\n", + " \n", + "
\n", + "

\n", + " Basic Stats\n", + "

\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "
Mean13278.07855
Minimum0.0
Maximum60000000.0
Zeros(%)
\n", + " \n", + "\n", + "
\n", + "
\n", + "

Frequency

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
ValueCountFrecuency (%)
1.31710.374%
1.21400.306%
1.41380.302%
None1310.287%
2.11300.284%
2.41260.276%
1.61200.262%
0.51190.26%
1.11160.254%
3.81140.249%
\"Missing\"00.0%
\n", + "
\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "

Quantile statistics

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Minimum0.0
5-th percentile
Q1
Median
Q3
95-th percentile
Maximum60000000.0
Range60000000.0
Interquartile range0.0
\n", + "
\n", + "
\n", + "

Descriptive statistics

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Standard deviation574988.87641
Coef of variation43.30362
Kurtosis6796.17061
Mean13278.07855
MAD0.0
Skewness
Sum605281210.638
Variance330612207995.7785
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "
Viewing 10 of 45716 rows / 10 columns
\n", + "
32 partition(s)
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "
name
\n", + "
1 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
id
\n", + "
2 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
nametype
\n", + "
3 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
recclass
\n", + "
4 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
mass (g)
\n", + "
5 (double)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
fall
\n", + "
6 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
year
\n", + "
7 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
reclat
\n", + "
8 (double)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
reclong
\n", + "
9 (double)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + " 		\n", + "
GeoLocation
\n", + "
10 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
Acfer⋅232\n", + "
\n", + " 		\n", + "
240\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
H5\n", + "
\n", + " 		\n", + "
725.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1991⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
27.73944\n", + "
\n", + " 		\n", + "
4.32833\n", + "
\n", + " 		\n", + "
(27.739440,⋅4.328330)\n", + "
\n", + "
\n", + "
Asuka⋅87197\n", + "
\n", + " 		\n", + "
2554\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
H4\n", + "
\n", + " 		\n", + "
124.99\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1987⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
-72.0\n", + "
\n", + " 		\n", + "
26.0\n", + "
\n", + " 		\n", + "
(-72.000000,⋅26.000000)\n", + "
\n", + "
\n", + "
Gladstone⋅(iron)\n", + "
\n", + " 		\n", + "
10920\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
Iron,⋅IAB-MG\n", + "
\n", + " 		\n", + "
736600.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1915⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
-23.9\n", + "
\n", + " 		\n", + "
151.3\n", + "
\n", + " 		\n", + "
(-23.900000,⋅151.300000)\n", + "
\n", + "
\n", + "
Nullarbor⋅015\n", + "
\n", + " 		\n", + "
17955\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
L6\n", + "
\n", + " 		\n", + "
3986.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1980⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
Lewis⋅Cliff⋅86533\n", + "
\n", + " 		\n", + "
13461\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
H5\n", + "
\n", + " 		\n", + "
15.7\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1986⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
-84.26756\n", + "
\n", + " 		\n", + "
161.3631\n", + "
\n", + " 		\n", + "
(-84.267560,⋅161.363100)\n", + "
\n", + "
\n", + "
Grove⋅Mountains⋅053589\n", + "
\n", + " 		\n", + "
48447\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
L5\n", + "
\n", + " 		\n", + "
1.4\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/2006⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
-72.7825\n", + "
\n", + " 		\n", + "
75.300278\n", + "
\n", + " 		\n", + "
(-72.782500,⋅75.300278)\n", + "
\n", + "
\n", + "
Sayh⋅al⋅Uhaymir⋅108\n", + "
\n", + " 		\n", + "
23300\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
H6\n", + "
\n", + " 		\n", + "
16.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/2001⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
21.06667\n", + "
\n", + " 		\n", + "
57.31667\n", + "
\n", + " 		\n", + "
(21.066670,⋅57.316670)\n", + "
\n", + "
\n", + "
Northwest⋅Africa⋅3088\n", + "
\n", + " 		\n", + "
31218\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
L6\n", + "
\n", + " 		\n", + "
171.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/2003⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + " 		\n", + "
None\n", + "
\n", + "
\n", + "
Reckling⋅Peak⋅92423\n", + "
\n", + " 		\n", + "
22432\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
L6\n", + "
\n", + " 		\n", + "
3.8\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1992⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
-76.22029\n", + "
\n", + " 		\n", + "
158.37967\n", + "
\n", + " 		\n", + "
(-76.220290,⋅158.379670)\n", + "
\n", + "
\n", + "
Sweetwater\n", + "
\n", + " 		\n", + "
23770\n", + "
\n", + " 		\n", + "
Valid\n", + "
\n", + " 		\n", + "
H5\n", + "
\n", + " 		\n", + "
1760.0\n", + "
\n", + " 		\n", + "
Found\n", + "
\n", + " 		\n", + "
01/01/1961⋅12:00:00⋅AM\n", + "
\n", + " 		\n", + "
32.55\n", + "
\n", + " 		\n", + "
-100.41667\n", + "
\n", + " 		\n", + "
(32.550000,⋅-100.416670)\n", + "
\n", + "
\n", + "\n", + "\n", + "
Viewing 10 of 45716 rows / 10 columns
\n", + "
32 partition(s)
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "op.profiler.run(df, \"mass (g)\", infer=False)" ] }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading page (1/2)\n", - "Rendering (2/2) \n", - "Done \n" - ] - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "new_df.table_image(\"images/table2.png\")" - ] - }, - { - "cell_type": "markdown", + "execution_count": 41, "metadata": { - "lines_to_next_cell": 0 + "scrolled": false }, - "source": [ - "Note that you can use Optimus functions and Spark functions(`.WithColumn()`) and all the df function availables in a Spark Dataframe at the same time. To know about all the Optimus functionality please go to this [notebooks](examples/)\n", - "\n", - "### Custom functions\n", - "Spark have multiple ways to transform your data like rdd, Column Expression ,udf and pandas udf. In Optimus we create the `apply()` and `apply_expr` which handle all the implementation complexity.\n", - "\n", - "Here you apply a function to the \"billingid\" column. Sum 1 and 2 to the current column value. All powered by Pandas UDF" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading page (1/2)\n", - "Rendering (2/2) \n", - "Done \n" - ] - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def func(value, args):\n", - " return value + args[0] + args[1]\n", - "\n", - "df.cols.apply(\"height(ft)\",func,\"int\", [1,2]).table_image(\"images/table3.png\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you want to apply a Column Expression use `apply_expr()` like this. In this case we pasa an argument 10 to divide the actual column value" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[===================> ] 32%\r", + "Warning: Failed to load file:///tmp/optimus/styles/styles.css (ignore)\n", + "[=====================> ] 36%\r", + "[=========================> ] 43%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -630,71 +1970,12 @@ } ], "source": [ - "from pyspark.sql import functions as F\n", - "\n", - "def func(col_name, args):\n", - " return F.col(col_name)/20\n", - "\n", - "df.cols.apply(\"height(ft)\", func=func, args=20).table_image(\"images/table4.png\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can change the table output back to ascii if you which" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "op.output(\"ascii\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To return to HTML just:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "op.output(\"html\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Data profiling\n", - "\n", - "Optimus comes with a powerful and unique data profiler. Besides basic and advance stats like min, max, kurtosis, mad etc, \n", - "it also let you know what type of data has every column. For example if a string column have string, integer, float, bool, date Optimus can give you an unique overview about your data. \n", - "Just run `df.profile(\"*\")` to profile all the columns. For more info about the profiler please go to this [notebook](examples/new-api-profiler.ipynb).\n", - "\n", - "Let's load a \"big\" dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 115, - "metadata": {}, - "outputs": [], - "source": [ - "df = op.load.csv(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/Meteorite_Landings.csv\").h_repartition()" + "op.profiler.to_image(path=\"images/profiler_numeric.png\")" ] }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -739,7 +2020,7 @@ " \n", " \n", " Total size in memory\n", - " 44.6 MB\n", + " -1 Bytes\n", "\n", " \n", " \n", @@ -983,7 +2264,7 @@ " \n", "\n", "
\n", - " \n", + " \n", "
\n", " \n", " \n", @@ -992,7 +2273,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -1784,7 +3065,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -1792,8 +3073,17 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", - "Warning: Failed to load file:///C:/Users/ARGENI~1/AppData/Local/Temp/optimus/styles/styles.css (ignore)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[===================> ] 33%\r", + "Warning: Failed to load file:///tmp/optimus/styles/styles.css (ignore)\n", + "[======================> ] 37%\r", + "[==========================> ] 44%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -1825,7 +3115,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1869,7 +3159,7 @@ " \n", " \n", " Total size in memory\n", - " 41.7 MB\n", + " -1 Bytes\n", "\n", " \n", " \n", @@ -2112,7 +3402,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2123,7 +3413,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2134,7 +3424,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2145,7 +3435,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2156,7 +3446,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2167,7 +3457,7 @@ " \n", " \n", "
\n", - " \n", + " \n", "
\n", "\n", " \n", @@ -2954,7 +4244,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -2962,8 +4252,19 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", - "Warning: Failed to load file:///C:/Users/ARGENI~1/AppData/Local/Temp/optimus/styles/styles.css (ignore)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==================> ] 31%\r", + "Warning: Failed to load file:///tmp/optimus/styles/styles.css (ignore)\n", + "[=====================> ] 36%\r", + "[=======================> ] 39%\r", + "[========================> ] 41%\r", + "[===========================> ] 45%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -2994,7 +4295,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -3004,7 +4305,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -3013,7 +4314,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -3022,7 +4323,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -3031,14 +4332,14 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 46, "metadata": { "lines_to_next_cell": 0 }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAD11JREFUeJzt3X+s3XV9x/Hn695eUqhiS7kaVn5ctrJZ6TI1N0ZLN20x2SzL0EyDbFNm7tI0hsrWLbPu/qHL1qUms4506yZ4XUhmSxVFiDQq02JScMTbwRxw56gUC1j1GkFZoXrbfvbHPSVFW++5595zT/s5z0fS3Hu+5/vtefefZ7/5nHO+35RSkCSd+Xo6PYAkaXYYdEmqhEGXpEoYdEmqhEGXpEoYdEmqhEGXpEoYdEmqhEGXpEoYdFUvycYk30rybJJHkrytsb03yUeS/CDJ/iTXJylJ5jWef1mSkSQHkzyV5G+T9Hb2XyOd2rxODyDNgW8Bvwl8F3gH8G9JlgJXA28BXg0cAj79M8fdAnwPWAosAD4PPAF8bG7GlqYnXstF3SbJg8AHgRuAnaWUjzW2vxm4G+gDFgMHgIWllOcbz18LrC2lrOrI4NIUPENX9ZK8G9gADDQ2vQQ4H/glJs+4jzvx90uYDPvBJMe39fzMPtJpxaCrakkuAW4GrgS+Vko52jhDD3AQuPCE3S864fcngJ8A55dSjszVvNJM+KaoarcAKMA4QJL3AMsbz30KuCHJkiQLgfcfP6iUchD4EvCRJOcm6UnyK0neOLfjS80z6KpaKeUR4CPA15h8g/PXgXsbT9/MZLS/ATwA7AKOAEcbz78bOAt4BHgauA24YK5ml6bLN0WlhiRvAf6llHJJp2eRWuEZurpWkrOTrEkyL8kSJj/5cnun55Ja5Rm6ulaSc4CvAq8EngfuAm4opfy4o4NJLTLoklQJl1wkqRJz+jn0888/vwwMDMzlS0rSGW/v3r0/KKX0T7XfnAZ9YGCA0dHRuXxJSTrjJfl2M/u55CJJlTDoklQJgy5JlTDoklQJgy5JlWgq6En+LMnDSR5KsiPJ/CSXJrk/yaNJdiY5q93DSrNtx44dLF++nN7eXpYvX86OHTs6PZLUsimD3rjGxfuAwVLKcqAXeCfwYeCjpZTLmLwS3VA7B5Vm244dOxgeHmbr1q0cPnyYrVu3Mjw8bNR1xmp2yWUecHbj5rnnMHljgNVMXk4UJu+9+NbZH09qn02bNjEyMsKqVavo6+tj1apVjIyMsGnTpk6PJrVkyqCXUp4C/p7J+yseBH4E7AWeOeFOLk8CS052fJK1SUaTjI6Pj8/O1NIsGBsbY+XKlS/atnLlSsbGxjo0kTQzzSy5LGLy7uiXMnkPxgVM3in9Z530Kl+llJtKKYOllMH+/im/uSrNmWXLlrFnz54XbduzZw/Lli3r0ETSzDSz5PJmYH8pZbyUMgF8FlgBLGwswcDkfRm/06YZpbYYHh5maGiI3bt3MzExwe7duxkaGmJ4eLjTo0ktaeZaLgeA1zeuHf08kzfbHQV2A28HbgWuA+5o15BSO1x77bUArF+/nrGxMZYtW8amTZte2C6daZq6HnqSvwauYfJ+iw8Af8LkmvmtwHmNbX9USvnJL/p7BgcHixfnkqTpSbK3lDI41X5NXW2xlPJBJm/PdaLHgNe1MJskqQ38pqi6ml8sUk3m9Hro0unk+BeLRkZGWLlyJXv27GFoaPL7ca6j60w0p/cUdQ1dp5Ply5ezdetWVq1a9cK23bt3s379eh566KEOTia9WLNr6AZdXau3t5fDhw/T19f3wraJiQnmz5/P0aNHOziZ9GLNBt01dHUtv1ik2riGrq41PDzMNddcw4IFCzhw4AAXX3wxhw4d4sYbb+z0aFJLPEOXgLlcepTaxaCra23atImdO3eyf/9+jh07xv79+9m5c6dXW9QZyzdF1bV8U1RnCt8Ulabgm6KqjUFX1/Jqi6qNn3JR1/Jqi6qNZ+jqavfddx/79u3j2LFj7Nu3j/vuu6/TI0ktM+jqWuvXr2fbtm0sWrSInp4eFi1axLZt21i/fn2nR5Na4qdc1LX6+vro7e3l2LFjTExM0NfXR09PD0ePHmViYqLT40kv8FMu0hSOHDnCxMQEmzdv5tChQ2zevJmJiQmOHDky9cHSacigq6utWbOGDRs2cM4557BhwwbWrFnT6ZGklhl0dbVdu3axZcsWnnvuObZs2cKuXbs6PZLUMtfQ1bVcQ9eZwjV0aQrr1q1jYmKCxYsX09PTw+LFi5mYmGDdunWdHk1qiV8sUtfaunUrADfffDPHjh3j6aef5r3vfe8L26UzjWfo6morVqxg6dKl9PT0sHTpUlasWNHpkaSWeYauruVNolUb3xRV1/Im0TpTeJNoaQpeD11nCj/lIk3B66GrNq6hq0pJmtpv9erVMzree5HqdOIZuqpUSmnqz/bt27n88sshPVx++eVs37696WONuU43rqFLwMDGu3h881WdHkM6KdfQJanLGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqkRTQU+yMMltSf4nyViSNyQ5L8ndSR5t/FzU7mElSafW7Bn6jcAXSimvBH4DGAM2Al8upVwGfLnxWJLUIVMGPcm5wG8BIwCllJ+WUp4BrgZuaex2C/DWdg0pSZpaM2fovwyMA/+a5IEkH0+yAHhFKeUgQOPny092cJK1SUaTjI6Pj8/a4JKkF2sm6POA1wL/XEp5DXCIaSyvlFJuKqUMllIG+/v7WxxTkjSVZoL+JPBkKeX+xuPbmAz895JcAND4+f32jChJasaUQS+lfBd4IsmvNTZdCTwC3Alc19h2HXBHWyaUJDWl2TsWrQc+meQs4DHgPUz+Z/CpJEPAAeAd7RlRktSMpoJeSnkQONnF1a+c3XEkSa3ym6KSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVImmg56kN8kDST7feHxpkvuTPJpkZ5Kz2jemJGkq0zlDvwEYO+Hxh4GPllIuA54GhmZzMEnS9DQV9CQXAlcBH288DrAauK2xyy3AW9sxoCSpOc2eof8D8JfAscbjxcAzpZQjjcdPAktOdmCStUlGk4yOj4/PaFhJ0qlNGfQkvwt8v5Sy98TNJ9m1nOz4UspNpZTBUspgf39/i2NKkqYyr4l9rgB+L8kaYD5wLpNn7AuTzGucpV8IfKd9Y6qbXbH5Kzz1zPNtf52BjXe19e9fsvBs7t24uq2voe42ZdBLKR8APgCQ5E3AX5RS/jDJp4G3A7cC1wF3tHFOdbGnnnmexzdf1ekxZqzd/2FIM/kc+vuBDUn2MbmmPjI7I0mSWtHMkssLSin3APc0fn8MeN3sjyRJaoXfFJWkShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SarEvE4PIE3l8fl/AB/q9BQz9/h8gB91egxVzKDrtDdweDuPb76q02PM2MDGu3i800Ooai65SFIlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1Ilpgx6kouS7E4yluThJDc0tp+X5O4kjzZ+Lmr/uJKkU2nm8rlHgD8vpfxnkpcCe5PcDfwx8OVSyuYkG4GNwPvbN6q61ZKFZzOw8a5OjzFjSxae3ekRVLkpg15KOQgcbPz+bJIxYAlwNfCmxm63APdg0NUG925c3fbXGNh4VxXXXFd3m9YaepIB4DXA/cArGrE/Hv2Xn+KYtUlGk4yOj4/PbFpJ0ik1HfQkLwE+A/xpKeXHzR5XSrmplDJYShns7+9vZUZJUhOaCnqSPiZj/slSymcbm7+X5ILG8xcA32/PiJKkZjTzKZcAI8BYKWXLCU/dCVzX+P064I7ZH0+S1KxmPuVyBfAu4L+TPNjY9lfAZuBTSYaAA8A72jOiJKkZzXzKZQ+QUzx95eyOI0lqld8UlaRKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKzCjoSX4nyTeT7EuycbaGkiRNX8tBT9IL/BPwFuBVwLVJXjVbg0mSpmcmZ+ivA/aVUh4rpfwUuBW4enbGkiRN10yCvgR44oTHTza2vUiStUlGk4yOj4/P4OUkSb/ITIKek2wrP7ehlJtKKYOllMH+/v4ZvJwk6ReZSdCfBC464fGFwHdmNo4kqVUzCfrXgcuSXJrkLOCdwJ2zM5YkabrmtXpgKeVIkuuBLwK9wCdKKQ/P2mSSpGlpOegApZRdwK5ZmkWSNAN+U1SSKmHQJakSBl2SKmHQJakSBl2SKmHQJakSBl2SKmHQJakSM/pikXS6Sk527bgpjvnw9F+nlJ+7Hp3UMQZdVTK06kYuuUhSJQy6JFXCoEtSJQy6JFXCoEtSJQy6JFXCoEtSJQy6JFUic/kFjCTjwLfn7AWl5p0P/KDTQ0incEkppX+qneY06NLpKsloKWWw03NIM+GSiyRVwqBLUiUMujTppk4PIM2Ua+iSVAnP0CWpEgZdkiph0CWpEgZdkiph0NU1knwuyd4kDydZ29g2lOR/k9yT5OYk/9jY3p/kM0m+3vhzRWenl6bmp1zUNZKcV0r5YZKzga8Dvw3cC7wWeBb4CvBfpZTrk2wHtpVS9iS5GPhiKWVZx4aXmuBNotVN3pfkbY3fLwLeBXy1lPJDgCSfBn618fybgVclOX7suUleWkp5di4HlqbDoKsrJHkTk5F+QynluST3AN8ETnXW3dPY9/m5mVCaOdfQ1S1eBjzdiPkrgdcD5wBvTLIoyTzg90/Y/0vA9ccfJHn1nE4rtcCgq1t8AZiX5BvA3wD/ATwF/B1wP/DvwCPAjxr7vw8YTPKNJI8A6+Z+ZGl6fFNUXS3JS0op/9c4Q78d+EQp5fZOzyW1wjN0dbsPJXkQeAjYD3yuw/NILfMMXZIq4Rm6JFXCoEtSJQy6JFXCoEtSJQy6JFXi/wHno0/sqV6NfgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD11JREFUeJzt3X+s3XV9x/Hn695eUqhiS7kaVn5ctrJZ6TI1N0ZLN20x2SzL0EyDbFNm7tI0hsrWLbPu/qHL1qUms4506yZ4XUhmSxVFiDQq02JScMTbwRxw56gUC1j1GkFZoXrbfvbHPSVFW++5595zT/s5z0fS3Hu+5/vtefefZ7/5nHO+35RSkCSd+Xo6PYAkaXYYdEmqhEGXpEoYdEmqhEGXpEoYdEmqhEGXpEoYdEmqhEGXpEoYdFUvycYk30rybJJHkrytsb03yUeS/CDJ/iTXJylJ5jWef1mSkSQHkzyV5G+T9Hb2XyOd2rxODyDNgW8Bvwl8F3gH8G9JlgJXA28BXg0cAj79M8fdAnwPWAosAD4PPAF8bG7GlqYnXstF3SbJg8AHgRuAnaWUjzW2vxm4G+gDFgMHgIWllOcbz18LrC2lrOrI4NIUPENX9ZK8G9gADDQ2vQQ4H/glJs+4jzvx90uYDPvBJMe39fzMPtJpxaCrakkuAW4GrgS+Vko52jhDD3AQuPCE3S864fcngJ8A55dSjszVvNJM+KaoarcAKMA4QJL3AMsbz30KuCHJkiQLgfcfP6iUchD4EvCRJOcm6UnyK0neOLfjS80z6KpaKeUR4CPA15h8g/PXgXsbT9/MZLS/ATwA7AKOAEcbz78bOAt4BHgauA24YK5ml6bLN0WlhiRvAf6llHJJp2eRWuEZurpWkrOTrEkyL8kSJj/5cnun55Ja5Rm6ulaSc4CvAq8EngfuAm4opfy4o4NJLTLoklQJl1wkqRJz+jn0888/vwwMDMzlS0rSGW/v3r0/KKX0T7XfnAZ9YGCA0dHRuXxJSTrjJfl2M/u55CJJlTDoklQJgy5JlTDoklQJgy5JlWgq6En+LMnDSR5KsiPJ/CSXJrk/yaNJdiY5q93DSrNtx44dLF++nN7eXpYvX86OHTs6PZLUsimD3rjGxfuAwVLKcqAXeCfwYeCjpZTLmLwS3VA7B5Vm244dOxgeHmbr1q0cPnyYrVu3Mjw8bNR1xmp2yWUecHbj5rnnMHljgNVMXk4UJu+9+NbZH09qn02bNjEyMsKqVavo6+tj1apVjIyMsGnTpk6PJrVkyqCXUp4C/p7J+yseBH4E7AWeOeFOLk8CS052fJK1SUaTjI6Pj8/O1NIsGBsbY+XKlS/atnLlSsbGxjo0kTQzzSy5LGLy7uiXMnkPxgVM3in9Z530Kl+llJtKKYOllMH+/im/uSrNmWXLlrFnz54XbduzZw/Lli3r0ETSzDSz5PJmYH8pZbyUMgF8FlgBLGwswcDkfRm/06YZpbYYHh5maGiI3bt3MzExwe7duxkaGmJ4eLjTo0ktaeZaLgeA1zeuHf08kzfbHQV2A28HbgWuA+5o15BSO1x77bUArF+/nrGxMZYtW8amTZte2C6daZq6HnqSvwauYfJ+iw8Af8LkmvmtwHmNbX9USvnJL/p7BgcHixfnkqTpSbK3lDI41X5NXW2xlPJBJm/PdaLHgNe1MJskqQ38pqi6ml8sUk3m9Hro0unk+BeLRkZGWLlyJXv27GFoaPL7ca6j60w0p/cUdQ1dp5Ply5ezdetWVq1a9cK23bt3s379eh566KEOTia9WLNr6AZdXau3t5fDhw/T19f3wraJiQnmz5/P0aNHOziZ9GLNBt01dHUtv1ik2riGrq41PDzMNddcw4IFCzhw4AAXX3wxhw4d4sYbb+z0aFJLPEOXgLlcepTaxaCra23atImdO3eyf/9+jh07xv79+9m5c6dXW9QZyzdF1bV8U1RnCt8Ulabgm6KqjUFX1/Jqi6qNn3JR1/Jqi6qNZ+jqavfddx/79u3j2LFj7Nu3j/vuu6/TI0ktM+jqWuvXr2fbtm0sWrSInp4eFi1axLZt21i/fn2nR5Na4qdc1LX6+vro7e3l2LFjTExM0NfXR09PD0ePHmViYqLT40kv8FMu0hSOHDnCxMQEmzdv5tChQ2zevJmJiQmOHDky9cHSacigq6utWbOGDRs2cM4557BhwwbWrFnT6ZGklhl0dbVdu3axZcsWnnvuObZs2cKuXbs6PZLUMtfQ1bVcQ9eZwjV0aQrr1q1jYmKCxYsX09PTw+LFi5mYmGDdunWdHk1qiV8sUtfaunUrADfffDPHjh3j6aef5r3vfe8L26UzjWfo6morVqxg6dKl9PT0sHTpUlasWNHpkaSWeYauruVNolUb3xRV1/Im0TpTeJNoaQpeD11nCj/lIk3B66GrNq6hq0pJmtpv9erVMzree5HqdOIZuqpUSmnqz/bt27n88sshPVx++eVs37696WONuU43rqFLwMDGu3h881WdHkM6KdfQJanLGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqkRTQU+yMMltSf4nyViSNyQ5L8ndSR5t/FzU7mElSafW7Bn6jcAXSimvBH4DGAM2Al8upVwGfLnxWJLUIVMGPcm5wG8BIwCllJ+WUp4BrgZuaex2C/DWdg0pSZpaM2fovwyMA/+a5IEkH0+yAHhFKeUgQOPny092cJK1SUaTjI6Pj8/a4JKkF2sm6POA1wL/XEp5DXCIaSyvlFJuKqUMllIG+/v7WxxTkjSVZoL+JPBkKeX+xuPbmAz895JcAND4+f32jChJasaUQS+lfBd4IsmvNTZdCTwC3Alc19h2HXBHWyaUJDWl2TsWrQc+meQs4DHgPUz+Z/CpJEPAAeAd7RlRktSMpoJeSnkQONnF1a+c3XEkSa3ym6KSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVAmDLkmVMOiSVImmg56kN8kDST7feHxpkvuTPJpkZ5Kz2jemJGkq0zlDvwEYO+Hxh4GPllIuA54GhmZzMEnS9DQV9CQXAlcBH288DrAauK2xyy3AW9sxoCSpOc2eof8D8JfAscbjxcAzpZQjjcdPAktOdmCStUlGk4yOj4/PaFhJ0qlNGfQkvwt8v5Sy98TNJ9m1nOz4UspNpZTBUspgf39/i2NKkqYyr4l9rgB+L8kaYD5wLpNn7AuTzGucpV8IfKd9Y6qbXbH5Kzz1zPNtf52BjXe19e9fsvBs7t24uq2voe42ZdBLKR8APgCQ5E3AX5RS/jDJp4G3A7cC1wF3tHFOdbGnnnmexzdf1ekxZqzd/2FIM/kc+vuBDUn2MbmmPjI7I0mSWtHMkssLSin3APc0fn8MeN3sjyRJaoXfFJWkShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SaqEQZekShh0SarEvE4PIE3l8fl/AB/q9BQz9/h8gB91egxVzKDrtDdweDuPb76q02PM2MDGu3i800Ooai65SFIlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1IlDLokVcKgS1Ilpgx6kouS7E4yluThJDc0tp+X5O4kjzZ+Lmr/uJKkU2nm8rlHgD8vpfxnkpcCe5PcDfwx8OVSyuYkG4GNwPvbN6q61ZKFZzOw8a5OjzFjSxae3ekRVLkpg15KOQgcbPz+bJIxYAlwNfCmxm63APdg0NUG925c3fbXGNh4VxXXXFd3m9YaepIB4DXA/cArGrE/Hv2Xn+KYtUlGk4yOj4/PbFpJ0ik1HfQkLwE+A/xpKeXHzR5XSrmplDJYShns7+9vZUZJUhOaCnqSPiZj/slSymcbm7+X5ILG8xcA32/PiJKkZjTzKZcAI8BYKWXLCU/dCVzX+P064I7ZH0+S1KxmPuVyBfAu4L+TPNjY9lfAZuBTSYaAA8A72jOiJKkZzXzKZQ+QUzx95eyOI0lqld8UlaRKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKGHRJqoRBl6RKzCjoSX4nyTeT7EuycbaGkiRNX8tBT9IL/BPwFuBVwLVJXjVbg0mSpmcmZ+ivA/aVUh4rpfwUuBW4enbGkiRN10yCvgR44oTHTza2vUiStUlGk4yOj4/P4OUkSb/ITIKek2wrP7ehlJtKKYOllMH+/v4ZvJwk6ReZSdCfBC464fGFwHdmNo4kqVUzCfrXgcuSXJrkLOCdwJ2zM5YkabrmtXpgKeVIkuuBLwK9wCdKKQ/P2mSSpGlpOegApZRdwK5ZmkWSNAN+U1SSKmHQJakSBl2SKmHQJakSBl2SKmHQJakSBl2SKmHQJakSM/pikXS6Sk527bgpjvnw9F+nlJ+7Hp3UMQZdVTK06kYuuUhSJQy6JFXCoEtSJQy6JFXCoEtSJQy6JFXCoEtSJQy6JFUic/kFjCTjwLfn7AWl5p0P/KDTQ0incEkppX+qneY06NLpKsloKWWw03NIM+GSiyRVwqBLUiUMujTppk4PIM2Ua+iSVAnP0CWpEgZdkiph0CWpEgZdkiph0NU1knwuyd4kDydZ29g2lOR/k9yT5OYk/9jY3p/kM0m+3vhzRWenl6bmp1zUNZKcV0r5YZKzga8Dvw3cC7wWeBb4CvBfpZTrk2wHtpVS9iS5GPhiKWVZx4aXmuBNotVN3pfkbY3fLwLeBXy1lPJDgCSfBn618fybgVclOX7suUleWkp5di4HlqbDoKsrJHkTk5F+QynluST3AN8ETnXW3dPY9/m5mVCaOdfQ1S1eBjzdiPkrgdcD5wBvTLIoyTzg90/Y/0vA9ccfJHn1nE4rtcCgq1t8AZiX5BvA3wD/ATwF/B1wP/DvwCPAjxr7vw8YTPKNJI8A6+Z+ZGl6fFNUXS3JS0op/9c4Q78d+EQp5fZOzyW1wjN0dbsPJXkQeAjYD3yuw/NILfMMXZIq4Rm6JFXCoEtSJQy6JFXCoEtSJQy6JFXi/wHno0/sqV6NfgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -3055,14 +4356,21 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 47, "metadata": { "lines_to_next_cell": 0 }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "object of type cannot be safely interpreted as an integer.\n" + ] + }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAADt9JREFUeJzt3X+MZWddx/H3ZwYKia2Y2Ahld5FGtmKtRmBTTWqkCpVtNSx/oNkaRUjD+IfLD1FCBa1YIxFIwJg0yBg3/IhSEQ1OYLEkCEELhV1+pGG3qUyqoeNiaqSiBBF25usfc9u9DDP33Nm997l3z75fm5Pc82Oe+3TSfPa7z3nOc1JVSJLaWJh1ByTpYmLoSlJDhq4kNWToSlJDhq4kNWToSlJDhq4k7SDJ0SQPJfnCDueT5E+SrCa5N8kzu9o0dCVpZ+8ADo44fyOwf7AtAW/ratDQlaQdVNXHga+MuOQQ8K7adA/wPUmuGNXmYybZwe188Sef5yNvU/bB175u1l3ovfd/+t5Zd+Gi8LHXH8n5trGbzLnq7g//GpsV6iOWq2p5F1+3B3hwaH9tcOzLO/3A1ENXkubVIGB3E7JbbfeXxMjQN3Ql9UuajpquAfuG9vcCp0f9gGO6knoliwtjbxOwArxoMIvhJ4CvVtWOQwtgpSupbyZY6SZ5D3A9cHmSNeD3gMcCVNWfAseAm4BV4OvAS7raNHQl9UvO+17co6rq5o7zBfz6bto0dCX1y8LkQncaDF1JvZIJVrrTYOhK6peF+Z4fYOhK6hdDV5LaiaErSQ0ZupLUkDfSJKkdZy9IUkuLi7PuwUiGrqR+8eEISWrH4QVJaqnt0o67ZuhK6heHFySpHR+OkKSWDF1JasjQlaR2nL0gSS15I02SGnLKmCS1M6G3/E6NoSupXxzTlaSGnL0gSe34cIQkteTwgiQ1ZOhKUjtxEXNJashKV5Ia8ok0SWroQn8iLcnTgUPAHqCA08BKVd035b5J0q5lzivdkX8lJHkNcCcQ4NPA8cHn9yS5dfrdk6RdWlgYf5uBrkr3FuCHq+pbwweTvAU4CfzRdj+UZAlYArj9B67m8JP2TqCrktRt3h+O6OrdBvDkbY5fMTi3raparqoDVXXAwJXU1AQr3SQHk9yfZHW7f90neUqSjyb5XJJ7k9zU1WZXpftK4CNJvgg8ODj2FOBpwJHOHktSaxOaMpZkEbgDuAFYA44nWamqU0OX/Q7w3qp6W5KrgWPAU0e1OzJ0q+rvk1wFXMvmjbQ88uVVtX6u/zGSNDWTm6d7LbBaVQ9sNps72ZxUMBy6BXz34PMT2JxoMFLn7IWq2gDu2W1vJWkWJjimu4ez/8KHzYLzx7dc83rgw0leBnwX8NyuRud7xFmSdmtxYewtyVKSE0Pb0lBL25XMtWX/ZuAdVbUXuAl4dzJ6orAPR0jql10ML1TVMrC8w+k1YN/Q/l6+c/jgFuDgoK1PJnk8cDnw0E7faaUrqVeysDD21uE4sD/JlUkuAQ4DK1uu+RLwHIAkPwQ8HviPUY1a6Urqlwk9BlxVZ5IcAe4CFoGjVXUyye3AiapaAX4T+LMkv8Hm0MOLq2rrEMS3MXQl9csEHwOuqmNsTgMbPnbb0OdTwHW7adPQldQrcWlHSWrIRcwlqSErXUlqZ96XdjR0JfXLhb6IuSRdUBxekKSGHF6QpHayON+xNt+9k6TdstKVpHZ8OEKSWjJ0JamhOX8xpaErqVfm/W3Ahq6kfjF0Jakhx3QlqSErXUlqxwVvJKklhxckqZ24iLkkNWSlK0kNuZ6uJDXkjTRJascFbySpJStdSWroYl/E/IOvfd20v+Ki93Nv+MNZd6H3rnjzG2fdBY3J4QVJasnhBUlqyEpXkhpynq4ktZNFQ1eS2nFpR0lqx9kLktTSnFe68907SdqtZPyts6kcTHJ/ktUkt+5wzS8mOZXkZJK/7GrTSldSv0xonm6SReAO4AZgDTieZKWqTg1dsx/4beC6qno4yfd1tWvoSuqVLExsEfNrgdWqegAgyZ3AIeDU0DUvBe6oqocBquqhrkYdXpDULwsZe0uylOTE0LY01NIe4MGh/bXBsWFXAVcluTvJPUkOdnXPSldSv+zi4YiqWgaWd2ppux/Zsv8YYD9wPbAX+Mck11TVf+30nYaupF6Z4NuA14B9Q/t7gdPbXHNPVX0L+Jck97MZwsd3atThBUn9MrnZC8eB/UmuTHIJcBhY2XLN+4Gf3vzaXM7mcMMDoxq10pXUK5N6G3BVnUlyBLgLWASOVtXJJLcDJ6pqZXDuZ5OcAtaBV1fVf45q19CV1C8TfDiiqo4Bx7Ycu23ocwGvGmxjMXQl9YuPAUtSQy5iLkntxPV0JakhhxckqSEXMZekdhxekKSWvJEmSQ3N+SLmhq6kXvF1PZLUkpWuJDVk6EpSOxNc2nEqDF1J/eKUMUlqyBtpktSQwwuS1M6kFjGfFkNXUq/87+MfN/a1l02xHzuZ7xFnSeoZQ1eSGjrn0E3ykkl2RJIuBudT6f7+TieSLCU5keTEJz609Y3FknTxGnkjLcm9O50CnrjTz1XVMrAM8MfHPl7n3DtJ6pmu2QtPBJ4HPLzleIBPTKVHktRjXaH7AeDSqvr81hNJPjaVHklSj40M3aq6ZcS5X5p8dySp33w4QlKvfGvxsbPuwkiGrqReqTm/dW/oSuqV9Y2NWXdhJENXUq/UnJe6hq6kXtkwdCWpnTnPXENXUr/M+/CCq4xJ6pX12hh765LkYJL7k6wmuXXEdS9MUkkOdLVppSupVyY1pptkEbgDuAFYA44nWamqU1uuuwx4OfCpcdq10pXUKxsbNfbW4VpgtaoeqKpvAncCh7a57g+ANwHfGKd/hq6kXqkafxtehnawLQ01tQd4cGh/bXDsUUmeAeyrqg+M2z+HFyT1ym5upA0vQ7uN7V4r/GjjSRaAtwIv3kX3DF1J/bLBxGYvrAH7hvb3AqeH9i8DrgE+lgTgScBKkudX1YmdGjV0JfXKxuQeAz4O7E9yJfBvwGHg0dUVq+qrwOWP7A+Wu/2tUYELhq6knum+PzaeqjqT5AhwF7AIHK2qk0luB05U1Tm9i8zQldQrk3w4oqqOAce2HLtth2uvH6dNQ1dSr8z7E2mGrqReccEbSWrI0JWkhlzEXJIamvNC19CV1C/eSJOkhhzTlaSGrHQlqaH1ST2SNiWGrqResdKVpIYc05WkhgxdSWrI4QVJasjQlaSGLvrZC+//9L3T/oqL3hVvfuOsu9B7z3z1a2bdhYvDP9113k1Y6UpSQxN8R9pUGLqSesVKV5IamvMhXUNXUr+sr7ueriQ14/CCJDXkjTRJashKV5IamvPMNXQl9YsL3khSQxu+DViS2rHSlaSGDF1JasjZC5LUkI8BS1JDVrqS1ND6nM9eWJh1ByRpkqrG37okOZjk/iSrSW7d5vyrkpxKcm+SjyT5/q42DV1JvVJVY2+jJFkE7gBuBK4Gbk5y9ZbLPgccqKofBd4HvKmrf4aupF7ZqBp763AtsFpVD1TVN4E7gUPDF1TVR6vq64Pde4C9XY0aupJ6ZTeVbpKlJCeGtqWhpvYADw7trw2O7eQW4ENd/fNGmqReObOLG2lVtQws73A62/3IthcmvwwcAJ7d9Z2GrqRemeCUsTVg39D+XuD01ouSPBd4HfDsqvq/rkYdXpDUKxOcvXAc2J/kyiSXAIeBleELkjwDeDvw/Kp6aJz+WelK6pVJrb1QVWeSHAHuAhaBo1V1MsntwImqWgHeDFwK/HUSgC9V1fNHtWvoSuqVST6RVlXHgGNbjt029Pm5u23T0JXUKz4GLEkNrRu6ktSO6+lKUkMOL0hSQxtzvqCuoSupV6x0Jakhx3QlqSFDV5IamvfQ7Vx7IcnTkzwnyaVbjh+cXrck6dxMahHzaRkZukleDvwd8DLgC0mGF/B9wzQ7JknnYoKLmE9FV6X7UuBZVfUC4Hrgd5O8YnBuu7UmN08MLQx8+jN3T6ankjSGSb4jbRq6Qnexqr4GUFX/ymbw3pjkLYwI3aparqoDVXXgyc+6blJ9laRO6xsbY2+z0BW6/57kxx7ZGQTwzwOXAz8yzY5J0rmY9+GFrtkLLwLODB+oqjPAi5K8fWq9kqRzdEE/HFFVayPOOVgrae5c0KErSReaOV96wdCV1C9WupLU0KxmJYzL0JXUK1a6ktSQY7qS1NBGObwgSc3M+eiCoSupXxzTlaSGnL0gSQ1Z6UpSQ85ekKSGrHQlqaENDF1JamZ93RtpktSMwwuS1JA30iSpoXmvdLvekSZJF5TaxZ8uSQ4muT/JapJbtzn/uCR/NTj/qSRP7WrT0JXUK5N6MWWSReAO4EbgauDmJFdvuewW4OGqehrwVuCNXf0zdCX1yvpGjb11uBZYraoHquqbwJ3AoS3XHALeOfj8PuA5STKqUUNXUq9U1dhbkqUkJ4a2paGm9gAPDu2vDY6x3TWDN6V/FfjeUf3zRpqkXtnNjbSqWgaWdzi9XcW6tfFxrvk2hq6kXukaq92FNWDf0P5e4PQO16wleQzwBOAroxp1eEFSr0zqRhpwHNif5MoklwCHgZUt16wAvzr4/ELgH6qj1LbSldQrk5qnW1VnkhwB7gIWgaNVdTLJ7cCJqloB/hx4d5JVNivcw13tGrqSemWMWQljq6pjwLEtx24b+vwN4Bd206ahK6lX5v2JNENXUq9M8EbaVBi6knrFSleSGprzzDV0JfXLvL8NOPNeis9CkqXBkyqaEn/H0+fveD75cMT2lrov0Xnydzx9/o7nkKErSQ0ZupLUkKG7PcfBps/f8fT5O55D3kiTpIasdCWpIUNXkhoydId0vflT5y/J0SQPJfnCrPvSV0n2JflokvuSnEzyiln3SWc5pjswePPnPwM3sLka/HHg5qo6NdOO9UySnwK+Bryrqq6ZdX/6KMkVwBVV9dkklwGfAV7g/8vzwUr3rHHe/KnzVFUfp+N1Jjo/VfXlqvrs4PP/APfxnS9U1IwYumeN8+ZP6YKS5KnAM4BPzbYneoShe9au3+opzbMklwJ/A7yyqv571v3RJkP3rHHe/CldEJI8ls3A/Yuq+ttZ90dnGbpnjfPmT2nuJQmbL0y8r6reMuv+6NsZugNVdQZ45M2f9wHvraqTs+1V/yR5D/BJ4AeTrCW5ZdZ96qHrgF8BfibJ5wfbTbPulDY5ZUySGrLSlaSGDF1JasjQlaSGDF1JasjQlaSGDF1JasjQlaSG/h8C9h458GSrxAAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADt9JREFUeJzt3X+MZWddx/H3ZwYKia2Y2Ahld5FGtmKtRmBTTWqkCpVtNSx/oNkaRUjD+IfLD1FCBa1YIxFIwJg0yBg3/IhSEQ1OYLEkCEELhV1+pGG3qUyqoeNiaqSiBBF25usfc9u9DDP33Nm997l3z75fm5Pc82Oe+3TSfPa7z3nOc1JVSJLaWJh1ByTpYmLoSlJDhq4kNWToSlJDhq4kNWToSlJDhq4k7SDJ0SQPJfnCDueT5E+SrCa5N8kzu9o0dCVpZ+8ADo44fyOwf7AtAW/ratDQlaQdVNXHga+MuOQQ8K7adA/wPUmuGNXmYybZwe188Sef5yNvU/bB175u1l3ovfd/+t5Zd+Gi8LHXH8n5trGbzLnq7g//GpsV6iOWq2p5F1+3B3hwaH9tcOzLO/3A1ENXkubVIGB3E7JbbfeXxMjQN3Ql9UuajpquAfuG9vcCp0f9gGO6knoliwtjbxOwArxoMIvhJ4CvVtWOQwtgpSupbyZY6SZ5D3A9cHmSNeD3gMcCVNWfAseAm4BV4OvAS7raNHQl9UvO+17co6rq5o7zBfz6bto0dCX1y8LkQncaDF1JvZIJVrrTYOhK6peF+Z4fYOhK6hdDV5LaiaErSQ0ZupLUkDfSJKkdZy9IUkuLi7PuwUiGrqR+8eEISWrH4QVJaqnt0o67ZuhK6heHFySpHR+OkKSWDF1JasjQlaR2nL0gSS15I02SGnLKmCS1M6G3/E6NoSupXxzTlaSGnL0gSe34cIQkteTwgiQ1ZOhKUjtxEXNJashKV5Ia8ok0SWroQn8iLcnTgUPAHqCA08BKVd035b5J0q5lzivdkX8lJHkNcCcQ4NPA8cHn9yS5dfrdk6RdWlgYf5uBrkr3FuCHq+pbwweTvAU4CfzRdj+UZAlYArj9B67m8JP2TqCrktRt3h+O6OrdBvDkbY5fMTi3raparqoDVXXAwJXU1AQr3SQHk9yfZHW7f90neUqSjyb5XJJ7k9zU1WZXpftK4CNJvgg8ODj2FOBpwJHOHktSaxOaMpZkEbgDuAFYA44nWamqU0OX/Q7w3qp6W5KrgWPAU0e1OzJ0q+rvk1wFXMvmjbQ88uVVtX6u/zGSNDWTm6d7LbBaVQ9sNps72ZxUMBy6BXz34PMT2JxoMFLn7IWq2gDu2W1vJWkWJjimu4ez/8KHzYLzx7dc83rgw0leBnwX8NyuRud7xFmSdmtxYewtyVKSE0Pb0lBL25XMtWX/ZuAdVbUXuAl4dzJ6orAPR0jql10ML1TVMrC8w+k1YN/Q/l6+c/jgFuDgoK1PJnk8cDnw0E7faaUrqVeysDD21uE4sD/JlUkuAQ4DK1uu+RLwHIAkPwQ8HviPUY1a6Urqlwk9BlxVZ5IcAe4CFoGjVXUyye3AiapaAX4T+LMkv8Hm0MOLq2rrEMS3MXQl9csEHwOuqmNsTgMbPnbb0OdTwHW7adPQldQrcWlHSWrIRcwlqSErXUlqZ96XdjR0JfXLhb6IuSRdUBxekKSGHF6QpHayON+xNt+9k6TdstKVpHZ8OEKSWjJ0JamhOX8xpaErqVfm/W3Ahq6kfjF0Jakhx3QlqSErXUlqxwVvJKklhxckqZ24iLkkNWSlK0kNuZ6uJDXkjTRJascFbySpJStdSWroYl/E/IOvfd20v+Ki93Nv+MNZd6H3rnjzG2fdBY3J4QVJasnhBUlqyEpXkhpynq4ktZNFQ1eS2nFpR0lqx9kLktTSnFe68907SdqtZPyts6kcTHJ/ktUkt+5wzS8mOZXkZJK/7GrTSldSv0xonm6SReAO4AZgDTieZKWqTg1dsx/4beC6qno4yfd1tWvoSuqVLExsEfNrgdWqegAgyZ3AIeDU0DUvBe6oqocBquqhrkYdXpDULwsZe0uylOTE0LY01NIe4MGh/bXBsWFXAVcluTvJPUkOdnXPSldSv+zi4YiqWgaWd2ppux/Zsv8YYD9wPbAX+Mck11TVf+30nYaupF6Z4NuA14B9Q/t7gdPbXHNPVX0L+Jck97MZwsd3atThBUn9MrnZC8eB/UmuTHIJcBhY2XLN+4Gf3vzaXM7mcMMDoxq10pXUK5N6G3BVnUlyBLgLWASOVtXJJLcDJ6pqZXDuZ5OcAtaBV1fVf45q19CV1C8TfDiiqo4Bx7Ycu23ocwGvGmxjMXQl9YuPAUtSQy5iLkntxPV0JakhhxckqSEXMZekdhxekKSWvJEmSQ3N+SLmhq6kXvF1PZLUkpWuJDVk6EpSOxNc2nEqDF1J/eKUMUlqyBtpktSQwwuS1M6kFjGfFkNXUq/87+MfN/a1l02xHzuZ7xFnSeoZQ1eSGjrn0E3ykkl2RJIuBudT6f7+TieSLCU5keTEJz609Y3FknTxGnkjLcm9O50CnrjTz1XVMrAM8MfHPl7n3DtJ6pmu2QtPBJ4HPLzleIBPTKVHktRjXaH7AeDSqvr81hNJPjaVHklSj40M3aq6ZcS5X5p8dySp33w4QlKvfGvxsbPuwkiGrqReqTm/dW/oSuqV9Y2NWXdhJENXUq/UnJe6hq6kXtkwdCWpnTnPXENXUr/M+/CCq4xJ6pX12hh765LkYJL7k6wmuXXEdS9MUkkOdLVppSupVyY1pptkEbgDuAFYA44nWamqU1uuuwx4OfCpcdq10pXUKxsbNfbW4VpgtaoeqKpvAncCh7a57g+ANwHfGKd/hq6kXqkafxtehnawLQ01tQd4cGh/bXDsUUmeAeyrqg+M2z+HFyT1ym5upA0vQ7uN7V4r/GjjSRaAtwIv3kX3DF1J/bLBxGYvrAH7hvb3AqeH9i8DrgE+lgTgScBKkudX1YmdGjV0JfXKxuQeAz4O7E9yJfBvwGHg0dUVq+qrwOWP7A+Wu/2tUYELhq6knum+PzaeqjqT5AhwF7AIHK2qk0luB05U1Tm9i8zQldQrk3w4oqqOAce2HLtth2uvH6dNQ1dSr8z7E2mGrqReccEbSWrI0JWkhlzEXJIamvNC19CV1C/eSJOkhhzTlaSGrHQlqaH1ST2SNiWGrqResdKVpIYc05WkhgxdSWrI4QVJasjQlaSGLvrZC+//9L3T/oqL3hVvfuOsu9B7z3z1a2bdhYvDP9113k1Y6UpSQxN8R9pUGLqSesVKV5IamvMhXUNXUr+sr7ueriQ14/CCJDXkjTRJashKV5IamvPMNXQl9YsL3khSQxu+DViS2rHSlaSGDF1JasjZC5LUkI8BS1JDVrqS1ND6nM9eWJh1ByRpkqrG37okOZjk/iSrSW7d5vyrkpxKcm+SjyT5/q42DV1JvVJVY2+jJFkE7gBuBK4Gbk5y9ZbLPgccqKofBd4HvKmrf4aupF7ZqBp763AtsFpVD1TVN4E7gUPDF1TVR6vq64Pde4C9XY0aupJ6ZTeVbpKlJCeGtqWhpvYADw7trw2O7eQW4ENd/fNGmqReObOLG2lVtQws73A62/3IthcmvwwcAJ7d9Z2GrqRemeCUsTVg39D+XuD01ouSPBd4HfDsqvq/rkYdXpDUKxOcvXAc2J/kyiSXAIeBleELkjwDeDvw/Kp6aJz+WelK6pVJrb1QVWeSHAHuAhaBo1V1MsntwImqWgHeDFwK/HUSgC9V1fNHtWvoSuqVST6RVlXHgGNbjt029Pm5u23T0JXUKz4GLEkNrRu6ktSO6+lKUkMOL0hSQxtzvqCuoSupV6x0Jakhx3QlqSFDV5IamvfQ7Vx7IcnTkzwnyaVbjh+cXrck6dxMahHzaRkZukleDvwd8DLgC0mGF/B9wzQ7JknnYoKLmE9FV6X7UuBZVfUC4Hrgd5O8YnBuu7UmN08MLQx8+jN3T6ankjSGSb4jbRq6Qnexqr4GUFX/ymbw3pjkLYwI3aparqoDVXXgyc+6blJ9laRO6xsbY2+z0BW6/57kxx7ZGQTwzwOXAz8yzY5J0rmY9+GFrtkLLwLODB+oqjPAi5K8fWq9kqRzdEE/HFFVayPOOVgrae5c0KErSReaOV96wdCV1C9WupLU0KxmJYzL0JXUK1a6ktSQY7qS1NBGObwgSc3M+eiCoSupXxzTlaSGnL0gSQ1Z6UpSQ85ekKSGrHQlqaENDF1JamZ93RtpktSMwwuS1JA30iSpoXmvdLvekSZJF5TaxZ8uSQ4muT/JapJbtzn/uCR/NTj/qSRP7WrT0JXUK5N6MWWSReAO4EbgauDmJFdvuewW4OGqehrwVuCNXf0zdCX1yvpGjb11uBZYraoHquqbwJ3AoS3XHALeOfj8PuA5STKqUUNXUq9U1dhbkqUkJ4a2paGm9gAPDu2vDY6x3TWDN6V/FfjeUf3zRpqkXtnNjbSqWgaWdzi9XcW6tfFxrvk2hq6kXukaq92FNWDf0P5e4PQO16wleQzwBOAroxp1eEFSr0zqRhpwHNif5MoklwCHgZUt16wAvzr4/ELgH6qj1LbSldQrk5qnW1VnkhwB7gIWgaNVdTLJ7cCJqloB/hx4d5JVNivcw13tGrqSemWMWQljq6pjwLEtx24b+vwN4Bd206ahK6lX5v2JNENXUq9M8EbaVBi6knrFSleSGprzzDV0JfXLvL8NOPNeis9CkqXBkyqaEn/H0+fveD75cMT2lrov0Xnydzx9/o7nkKErSQ0ZupLUkKG7PcfBps/f8fT5O55D3kiTpIasdCWpIUNXkhoydId0vflT5y/J0SQPJfnCrPvSV0n2JflokvuSnEzyiln3SWc5pjswePPnPwM3sLka/HHg5qo6NdOO9UySnwK+Bryrqq6ZdX/6KMkVwBVV9dkklwGfAV7g/8vzwUr3rHHe/KnzVFUfp+N1Jjo/VfXlqvrs4PP/APfxnS9U1IwYumeN8+ZP6YKS5KnAM4BPzbYneoShe9au3+opzbMklwJ/A7yyqv571v3RJkP3rHHe/CldEJI8ls3A/Yuq+ttZ90dnGbpnjfPmT2nuJQmbL0y8r6reMuv+6NsZugNVdQZ45M2f9wHvraqTs+1V/yR5D/BJ4AeTrCW5ZdZ96qHrgF8BfibJ5wfbTbPulDY5ZUySGrLSlaSGDF1JasjQlaSGDF1JasjQlaSGDF1JasjQlaSG/h8C9h458GSrxAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -3107,7 +4415,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 48, "metadata": {}, "outputs": [ { @@ -3115,7 +4423,14 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -3145,7 +4460,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -3153,7 +4468,14 @@ "output_type": "stream", "text": [ "Loading page (1/2)\n", + "[> ] 0%\r", + "[======> ] 10%\r", + "[==============================> ] 50%\r", + "[============================================================] 100%\r", "Rendering (2/2) \n", + "[> ] 0%\r", + "[===============> ] 25%\r", + "[============================================================] 100%\r", "Done \n" ] }, @@ -3185,7 +4507,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 50, "metadata": {}, "outputs": [ { @@ -3194,7 +4516,7 @@ "DataFrame[pclass: bigint, survived: bigint, name: string, sex: string, age: double, sibsp: bigint, parch: bigint, ticket: string, fare: double, cabin: string, embarked: string, boat: string, body: double, home_dest: string]" ] }, - "execution_count": 45, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -3253,7 +4575,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -3262,28 +4584,29 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 52, "metadata": {}, "outputs": [ { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a5732f7bad1a4fb6818fb04e1e352269", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(IntProgress(value=0, description='Processing...', max=19, style=ProgressStyle(description_width…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "ename": "Py4JJavaError", + "evalue": "An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 538.0 failed 1 times, most recent failure: Lost task 0.0 in stage 538.0 (TID 6754, localhost, executor driver): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 377, in main\n process()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 372, in process\n serializer.dump_stream(func(split_index, iterator), outfile)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 352, in func\n return f(iterator)\n File \"/home/argenis/Documents/Optimus/optimus/io/save.py\", line 184, in _mongo\n collection.insert_one(as_dict)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 700, in insert_one\n session=session),\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 614, in _insert\n bypass_doc_val, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 602, in _insert_one\n acknowledged, _insert_command, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1279, in _retryable_write\n with self._tmp_session(session) as s:\n File \"/home/argenis/anaconda3/lib/python3.7/contextlib.py\", line 112, in __enter__\n return next(self.gen)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1611, in _tmp_session\n s = self._ensure_session(session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1598, in _ensure_session\n return self.__start_session(True, causal_consistency=False)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1551, in __start_session\n server_session = self._get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1584, in _get_server_session\n return self._topology.get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 429, in get_server_session\n None)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 200, in _select_servers_loop\n self._error_message(selector))\npymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:452)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:588)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:571)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:406)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:891)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:121)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1889)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1877)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1876)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1876)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:926)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2110)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2059)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2048)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:737)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2126)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:363)\n\tat org.apache.spark.rdd.RDD.collect(RDD.scala:944)\n\tat org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:166)\n\tat org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 377, in main\n process()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 372, in process\n serializer.dump_stream(func(split_index, iterator), outfile)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 352, in func\n return f(iterator)\n File \"/home/argenis/Documents/Optimus/optimus/io/save.py\", line 184, in _mongo\n collection.insert_one(as_dict)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 700, in insert_one\n session=session),\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 614, in _insert\n bypass_doc_val, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 602, in _insert_one\n acknowledged, _insert_command, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1279, in _retryable_write\n with self._tmp_session(session) as s:\n File \"/home/argenis/anaconda3/lib/python3.7/contextlib.py\", line 112, in __enter__\n return next(self.gen)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1611, in _tmp_session\n s = self._ensure_session(session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1598, in _ensure_session\n return self.__start_session(True, causal_consistency=False)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1551, in __start_session\n server_session = self._get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1584, in _get_server_session\n return self._topology.get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 429, in get_server_session\n None)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 200, in _select_servers_loop\n self._error_message(selector))\npymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:452)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:588)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:571)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:406)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:891)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:121)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mPy4JJavaError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menrich\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mdf_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc_request\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcalls\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0;36m60\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperiod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m60\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_tries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/Optimus/optimus/enricher.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, df, func_request, func_response, return_type, calls, period, max_tries)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;31m# Load the dataframe data in the enricher\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0mcollection_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollection_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/Optimus/optimus/enricher.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, df)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollection_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert_many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"records\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mis_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmongo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mport\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdb_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollection_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 61\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"df must by a Spark Dataframe or Pandas Dataframe\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/Optimus/optimus/helpers/decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtimeit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 50\u001b[0m \u001b[0m_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlog_time\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/Optimus/optimus/io/save.py\u001b[0m in \u001b[0;36mmongo\u001b[0;34m(host, port, db_name, collection_name, parallelism)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmessages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_mongo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msave\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\u001b[0m in \u001b[0;36mcount\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1054\u001b[0m \"\"\"\n\u001b[0;32m-> 1055\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1056\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1057\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\u001b[0m in \u001b[0;36msum\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1044\u001b[0m \u001b[0;36m6.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1045\u001b[0m \"\"\"\n\u001b[0;32m-> 1046\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1048\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\u001b[0m in \u001b[0;36mfold\u001b[0;34m(self, zeroValue, op)\u001b[0m\n\u001b[1;32m 915\u001b[0m \u001b[0;31m# zeroValue provided to each partition is unique from the one provided\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 916\u001b[0m \u001b[0;31m# to the final reduce call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m \u001b[0mvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 918\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzeroValue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\u001b[0m in \u001b[0;36mcollect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 814\u001b[0m \"\"\"\n\u001b[1;32m 815\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mSCCallSiteSync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 816\u001b[0;31m \u001b[0msock_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonRDD\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollectAndServe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 817\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_load_from_socket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msock_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd_deserializer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1255\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1256\u001b[0m return_value = get_return_value(\n\u001b[0;32m-> 1257\u001b[0;31m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m 1258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1259\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 64\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 326\u001b[0m raise Py4JJavaError(\n\u001b[1;32m 327\u001b[0m \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m 329\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m raise Py4JError(\n", + "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 538.0 failed 1 times, most recent failure: Lost task 0.0 in stage 538.0 (TID 6754, localhost, executor driver): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 377, in main\n process()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 372, in process\n serializer.dump_stream(func(split_index, iterator), outfile)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 352, in func\n return f(iterator)\n File \"/home/argenis/Documents/Optimus/optimus/io/save.py\", line 184, in _mongo\n collection.insert_one(as_dict)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 700, in insert_one\n session=session),\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 614, in _insert\n bypass_doc_val, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 602, in _insert_one\n acknowledged, _insert_command, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1279, in _retryable_write\n with self._tmp_session(session) as s:\n File \"/home/argenis/anaconda3/lib/python3.7/contextlib.py\", line 112, in __enter__\n return next(self.gen)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1611, in _tmp_session\n s = self._ensure_session(session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1598, in _ensure_session\n return self.__start_session(True, causal_consistency=False)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1551, in __start_session\n server_session = self._get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1584, in _get_server_session\n return self._topology.get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 429, in get_server_session\n None)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 200, in _select_servers_loop\n self._error_message(selector))\npymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:452)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:588)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:571)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:406)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:891)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:121)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1889)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1877)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1876)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1876)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:926)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2110)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2059)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2048)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:737)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2126)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:363)\n\tat org.apache.spark.rdd.RDD.collect(RDD.scala:944)\n\tat org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:166)\n\tat org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 377, in main\n process()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/python/lib/pyspark.zip/pyspark/worker.py\", line 372, in process\n serializer.dump_stream(func(split_index, iterator), outfile)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 2499, in pipeline_func\n return func(split, prev_func(split, iterator))\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pyspark/rdd.py\", line 352, in func\n return f(iterator)\n File \"/home/argenis/Documents/Optimus/optimus/io/save.py\", line 184, in _mongo\n collection.insert_one(as_dict)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 700, in insert_one\n session=session),\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 614, in _insert\n bypass_doc_val, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/collection.py\", line 602, in _insert_one\n acknowledged, _insert_command, session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1279, in _retryable_write\n with self._tmp_session(session) as s:\n File \"/home/argenis/anaconda3/lib/python3.7/contextlib.py\", line 112, in __enter__\n return next(self.gen)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1611, in _tmp_session\n s = self._ensure_session(session)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1598, in _ensure_session\n return self.__start_session(True, causal_consistency=False)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1551, in __start_session\n server_session = self._get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/mongo_client.py\", line 1584, in _get_server_session\n return self._topology.get_server_session()\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 429, in get_server_session\n None)\n File \"/home/argenis/anaconda3/lib/python3.7/site-packages/pymongo/topology.py\", line 200, in _select_servers_loop\n self._error_message(selector))\npymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused\n\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:452)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:588)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRunner.scala:571)\n\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:406)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:891)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:945)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:121)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\n" ] } ], @@ -3309,31 +4632,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading page (1/2)\n", - "Rendering (2/2) \n", - "Done \n" - ] - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "df_result.table_image(\"images/table7.png\")" ] @@ -3365,7 +4666,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -3374,7 +4675,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -3395,31 +4696,9 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading page (1/2)\n", - "Rendering (2/2) \n", - "Done \n" - ] - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "df_predict.cols.select([\"label\",\"prediction\"]).table_image(\"images/table8.png\")" ] @@ -3492,13 +4771,13 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ "from shutil import copyfile\n", "output_file = \"README.md\"\n", - "copyfile(\"README_.md\", output_file)\n", + "copyfile(\"readme_.md\", output_file)\n", "\n", "import sys\n", "import fileinput\n", @@ -3509,13 +4788,14 @@ "jupytext_header = False\n", "flag_remove = False\n", "\n", + "remove = [\"load_ext\", \"autoreload\",\"import sys\",\"sys.path.append\"]\n", + "\n", "for i, line in enumerate(fileinput.input(output_file, inplace=1)):\n", " done= False\n", " try:\n", " # Remove some helper lines\n", " for r in remove:\n", " if re.search(r, line):\n", - " line.strip()\n", " done= True\n", " \n", " #Remove the post process code\n", @@ -3523,13 +4803,11 @@ " flag_remove = True\n", " \n", " if flag_remove is True:\n", - " line.strip()\n", " done = True \n", " \n", " \n", " # Remove jupytext header\n", " if jupytext_header is True:\n", - " line.strip()\n", " done = True\n", " \n", " if \"---\\n\" == line: \n", @@ -3542,37 +4820,50 @@ " chars_image=re.search(\".to_image\", line)\n", " \n", " if chars_table:\n", - " line.strip()\n", " print(line[0:int(chars_table.start())]+\".table()\")\n", "\n", " m = re.search(r'table_image\\(\"(.*?)\"\\)', line).group(1)\n", " if m:\n", " buffer = \"![](\"+ m + \")\" \n", " elif chars_image:\n", - "# line.strip()\n", " m = re.search(r'to_image\\(path=\"(.*?)\"\\)', line).group(1)\n", " if m:\n", " buffer = \"![](\"+ m + \")\" \n", " else:\n", " sys.stdout.write(line)\n", " \n", - " if \"```\\n\"==line and buffer:\n", - " \n", - "# print(\"```\")\n", + " if \"```\\n\"==line and buffer: \n", " print(buffer)\n", " buffer = None\n", " \n", - " \n", - " \n", " except Exception as e:\n", " print(e)\n", " \n", + "fileinput.close()\n", + "\n", + "\n", + "# Remove empyt python cells\n", + "flag = False\n", + "for i, line in enumerate(fileinput.input(output_file, inplace=1)):\n", + " \n", + " if re.search(\"```python\", line): \n", + " flag = True\n", + " elif re.search(\"```\", line) and flag is True:\n", + " flag=False\n", + " elif flag is True:\n", + " flag = False\n", + " print(\"```python\")\n", + " print(line,end=\"\")\n", + " else:\n", + " print(line, end=\"\")\n", + " \n", + " \n", "fileinput.close()" ] }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 55, "metadata": {}, "outputs": [ { @@ -3581,7 +4872,7 @@ "'images/profiler.png'" ] }, - "execution_count": 193, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } @@ -3617,9 +4908,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.1" + "version": "3.7.3" } }, "nbformat": 4, - "nbformat_minor": 4 + "nbformat_minor": 2 } diff --git a/README_.md b/readme_.md similarity index 96% rename from README_.md rename to readme_.md index a08b82800..f1285e935 100644 --- a/README_.md +++ b/readme_.md @@ -6,7 +6,7 @@ jupyter: extension: .md format_name: markdown format_version: '1.1' - jupytext_version: 1.1.1 + jupytext_version: 1.1.5 kernelspec: display_name: Python 3 language: python @@ -245,6 +245,16 @@ Let's load a "big" dataset df = op.load.csv("https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/Meteorite_Landings.csv").h_repartition() ``` +### Numeric + +```python +op.profiler.run(df, "mass (g)", infer=False) +``` + +```python +op.profiler.to_image(path="images/profiler_numeric.png") +``` + ```python op.profiler.run(df, "name", infer=False) ``` @@ -469,7 +479,7 @@ Process the readme file to remove header from jupytext, python comments and conv ```python from shutil import copyfile output_file = "README.md" -copyfile("README_.md", output_file) +copyfile("readme_.md", output_file) import sys import fileinput @@ -480,13 +490,14 @@ pattern = r'"([A-Za-z0-9_\./\\-]*)"' jupytext_header = False flag_remove = False +remove = ["load_ext", "autoreload","import sys","sys.path.append"] + for i, line in enumerate(fileinput.input(output_file, inplace=1)): done= False try: # Remove some helper lines for r in remove: if re.search(r, line): - line.strip() done= True #Remove the post process code @@ -494,13 +505,11 @@ for i, line in enumerate(fileinput.input(output_file, inplace=1)): flag_remove = True if flag_remove is True: - line.strip() done = True # Remove jupytext header if jupytext_header is True: - line.strip() done = True if "---\n" == line: @@ -513,31 +522,44 @@ for i, line in enumerate(fileinput.input(output_file, inplace=1)): chars_image=re.search(".to_image", line) if chars_table: - line.strip() print(line[0:int(chars_table.start())]+".table()") m = re.search(r'table_image\("(.*?)"\)', line).group(1) if m: buffer = "![]("+ m + ")" elif chars_image: -# line.strip() m = re.search(r'to_image\(path="(.*?)"\)', line).group(1) if m: buffer = "![]("+ m + ")" else: sys.stdout.write(line) - if "```\n"==line and buffer: - -# print("```") + if "```\n"==line and buffer: print(buffer) buffer = None - - except Exception as e: print(e) +fileinput.close() + + +# Remove empyt python cells +flag = False +for i, line in enumerate(fileinput.input(output_file, inplace=1)): + + if re.search("```python", line): + flag = True + elif re.search("```", line) and flag is True: + flag=False + elif flag is True: + flag = False + print("```python") + print(line,end="") + else: + print(line, end="") + + fileinput.close() ``` diff --git a/requirements-docs.txt b/requirements-docs.txt index 55252f366..50fc3b7e9 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,6 +1,6 @@ findspark==1.3.0 numpy==1.16.4 -pytest==4.4.1 +pytest==4.6.2 matplotlib==3.0.3 setuptools==41.0.1 ipython==7.5.0 \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index c8603c085..144470c6c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,4 @@ imgkit==1.0.1 -bson==0.5.8 mock==3.0.5 nose==1.3.7 pylint==2.3.1 @@ -9,7 +8,7 @@ python_dateutil==2.7.5 numpy==1.16.4 matplotlib==3.0.3 pyspark==2.4.1 -pytest==4.4.1 +pytest==4.6.2 findspark==1.3.0 nose==1.3.7 seaborn==0.9.0 @@ -27,11 +26,11 @@ h5py>=2.7.0 flask==1.0.2 ipython==7.5.0 humanize==0.5.1 -psutil==5.6.2 +psutil==5.6.3 pytest-cov==2.7.1 backoff==1.8.0 pymongo==3.8.0 -h2o-pysparkling-2.4==2.4.10 +h2o_pysparkling_2.4==2.4.12 ratelimit==2.2.1 tqdm==4.28.1 kombu==4.6.1 diff --git a/requirements.txt b/requirements.txt index 21bb8b77a..75f6ec3e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,4 @@ imgkit==1.0.1 -bson==0.5.8 -httplib2==0.13.0 -google-api-python-client==1.7.9 -apiclient==1.0.4 packaging==19.0 requests==2.21.0 tqdm==4.28.1 @@ -25,8 +21,8 @@ flask==1.0.2 ipython==7.5.0 ratelimit==2.2.1 humanize==0.5.1 -h2o-pysparkling-2.4==2.4.10 -psutil==5.6.2 +h2o-pysparkling-2.4==2.4.12 +psutil==5.6.3 backoff==1.8.0 pymongo==3.8.0 kombu==4.6.1 diff --git a/setup.py b/setup.py index 5399caca2..0708345cd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ -import re import sys from setuptools import setup, find_packages +import re # Get version without importing, which avoids dependency issues @@ -41,7 +41,7 @@ def readme(): author='Favio Vazquez and Argenis Leon', author_email='favio.vazquez@ironmussa.com', url='https://github.com/ironmussa/Optimus/', - download_url='https://github.com/ironmussa/Optimus/archive/2.2.51.tar.gz', + download_url='https://github.com/ironmussa/Optimus/archive/2.2.6.tar.gz', description=('Optimus is the missing framework for cleaning and pre-processing data in a distributed fashion with ' 'pyspark.'), long_description=readme(), @@ -68,5 +68,5 @@ def readme(): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ], - keywords=['datacleaner', 'apachespark', 'spark', 'pyspark', 'data-wrangling', 'data-cleansing','data-profiling'], + keywords=['datacleaner', 'apachespark', 'spark', 'pyspark', 'data-wrangling', 'data-cleansing', 'data-profiling'], ) diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index d82ba210b..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,38 +0,0 @@ -import findspark - -findspark.init() - -import pytest - -from pyspark.sql.session import SparkSession -from pyspark import SparkContext - - -@pytest.fixture(scope="session") -def spark_context(request): - """ fixture for creating a spark context - Args: - request: pytest.FixtureRequest object - """ - sc = SparkContext.getOrCreate() - request.addfinalizer(lambda: sc.stop()) - - return sc - - -@pytest.fixture(scope="session") -def spark_session(request): - """Fixture for creating a spark session.""" - - spark = (SparkSession - .builder - .enableHiveSupport() - .getOrCreate()) - request.addfinalizer(lambda: spark.stop()) - - return spark - - -def pytest_assertrepr_compare(config, op, left, right): - if op in ('==', '!='): - return ['{0} {1} {2}'.format(left, op, right)] diff --git a/tests/creator/creator.ipynb b/tests/creator/creator.ipynb index 38eb21684..00d48aa2f 100644 --- a/tests/creator/creator.ipynb +++ b/tests/creator/creator.ipynb @@ -32,7 +32,7 @@ "metadata": {}, "outputs": [], "source": [ - "sys.path.append(\"../..\")" + "# import pyspark as ps" ] }, { @@ -40,6 +40,40 @@ "execution_count": 4, "metadata": {}, "outputs": [], + "source": [ + "# ps.version.__version__ " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# sys.path.append(\"../..\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n", + " from collections import Callable\n" + ] + } + ], "source": [ "from optimus import Optimus\n", "from optimus.helpers.test import Test" @@ -47,108 +81,59 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:optimus:Just check that Spark and all necessary environments vars are present...\n", + "INFO:optimus:-----\n", + "INFO:optimus:SPARK_HOME=C:\\opt\\spark\\spark-2.4.0-bin-hadoop2.7\n", + "INFO:optimus:HADOOP_HOME=C:\\opt\\hadoop-2.7.7\\bin\n", + "INFO:optimus:PYSPARK_PYTHON=C:\\Users\\argenisleon\\Anaconda3\\python.exe\n", + "INFO:optimus:PYSPARK_DRIVER_PYTHON=jupyter\n", + "INFO:optimus:PYSPARK_SUBMIT_ARGS=--conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", + "INFO:optimus:JAVA_HOME=C:\\java\n", + "INFO:optimus:Pyarrow Installed\n", + "INFO:optimus:-----\n", + "INFO:optimus:Starting or getting SparkSession and SparkContext...\n", + "INFO:optimus:Spark Version:2.4.0\n", + "INFO:optimus:\n", + " ____ __ _ \n", + " / __ \\____ / /_(_)___ ___ __ _______\n", + " / / / / __ \\/ __/ / __ `__ \\/ / / / ___/\n", + " / /_/ / /_/ / /_/ / / / / / / /_/ (__ ) \n", + " \\____/ .___/\\__/_/_/ /_/ /_/\\__,_/____/ \n", + " /_/ \n", + " \n", + "INFO:optimus:Transform and Roll out...\n", + "INFO:optimus:Optimus successfully imported. Have fun :).\n", + "INFO:optimus:Config.ini not found\n" + ] + }, + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: 'C:\\\\Users\\\\argenisleon\\\\Anaconda3\\\\lib\\\\site-packages\\\\optimus//css//styles.css'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mOptimus\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmaster\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'local'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\optimus\\optimus.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, session, master, app_name, checkpoint, path, file_system, verbose, server, repositories, packages, jars, options, additional_options, queue_url, queue_exchange, queue_routing_key)\u001b[0m\n\u001b[0;32m 143\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 144\u001b[0m \u001b[1;31m#\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 145\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_load_css\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 146\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 147\u001b[0m \u001b[1;31m# Set global output as html\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\optimus\\optimus.py\u001b[0m in \u001b[0;36m_load_css\u001b[1;34m()\u001b[0m\n\u001b[0;32m 158\u001b[0m \u001b[0mpath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdirname\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mabspath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m__file__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 159\u001b[0m \u001b[0murl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpath\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"//css//styles.css\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 160\u001b[1;33m \u001b[0mstyles\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"utf8\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 161\u001b[0m \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m''\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mstyles\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 162\u001b[0m \u001b[0mprint_html\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'C:\\\\Users\\\\argenisleon\\\\Anaconda3\\\\lib\\\\site-packages\\\\optimus//css//styles.css'" + ] } ], "source": [ - "op = Optimus(master='local')" + "op = Optimus(master='local', verbose=True)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": { "lines_to_next_cell": 2 }, @@ -26813,7 +26798,9 @@ { "cell_type": "code", "execution_count": 18, - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stdout", @@ -26825,13 +26812,15 @@ ], "source": [ "# t.create(None, \"cols.years_between\", None, \"df\", date_col, \"yyyy/MM/dd\")\n", - "t.delete(None, \"cols.years_between\", None, \"df\", date_col, \"yyyy/MM/dd\")\n" + "t.delete(None, \"cols.years_between\", None, \"df\", date_col, \"yyyy/MM/dd\")" ] }, { "cell_type": "code", "execution_count": 19, - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stdout", @@ -26843,7 +26832,7 @@ ], "source": [ "# t.create(None, \"cols.years_between\", \"multiple_columns\", \"df\", [date_col, date_col_B], \"yyyy/MM/dd\")\n", - "t.delete(None, \"cols.years_between\", \"multiple_columns\", \"df\", [date_col, date_col_B], \"yyyy/MM/dd\")\n" + "t.delete(None, \"cols.years_between\", \"multiple_columns\", \"df\", [date_col, date_col_B], \"yyyy/MM/dd\")" ] }, { diff --git a/tests/creator/creator.py b/tests/creator/creator.py index a5cbab9c8..4df10cc24 100644 --- a/tests/creator/creator.py +++ b/tests/creator/creator.py @@ -21,12 +21,23 @@ import sys -sys.path.append("../..") +# + +# import pyspark as ps + +# + +# ps.version.__version__ +# - + + + +# + +# sys.path.append("../..") +# - from optimus import Optimus from optimus.helpers.test import Test -op = Optimus(master='local') +op = Optimus(master='local', verbose=True) # + import pandas as pd diff --git a/tests/sql/main.sql b/tests/sql/main.sql new file mode 100644 index 000000000..2152b2f76 --- /dev/null +++ b/tests/sql/main.sql @@ -0,0 +1,1010 @@ +create table main ( + id INT, + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + color VARCHAR(50), + nulls VARCHAR(50), + booleans VARCHAR(50), + words TEXT +); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (1, 'MacGuiness', 'amacguiness0@123-reg.co.uk', 'Male', 'Fuscia', null, true, 'ultrices phasellus id sapien in sapien iaculis congue vivamus metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (2, 'Davidovitch', 'adavidovitch1@jiathis.com', 'Male', 'Puce', null, true, 'commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (3, 'Diggar', 'cdiggar2@europa.eu', 'Male', 'Maroon', null, false, 'ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (4, 'Bryenton', 'jbryenton3@godaddy.com', 'Female', 'Pink', null, true, 'interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum ac lobortis vel dapibus at diam nam tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (5, 'Thornley', 'gthornley4@whitehouse.gov', 'Female', 'Fuscia', null, true, 'mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (6, 'O''Hickee', 'tohickee5@51.la', 'Female', 'Fuscia', null, true, 'porttitor id consequat in consequat ut nulla sed accumsan felis ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (7, 'Brimner', 'nbrimner6@wiley.com', 'Male', 'Aquamarine', null, false, 'orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (8, 'Lahiff', 'wlahiff7@msu.edu', 'Female', 'Green', null, false, 'at velit vivamus vel nulla eget eros elementum pellentesque quisque porta volutpat erat quisque erat eros'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (9, 'Tarbert', 'htarbert8@cam.ac.uk', 'Male', 'Violet', null, true, 'mi in porttitor pede justo eu massa donec dapibus duis at velit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (10, 'Harwick', 'eharwick9@blog.com', 'Male', 'Turquoise', null, true, 'egestas metus aenean fermentum donec ut mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (11, 'Bachelar', 'mbachelara@1688.com', 'Male', 'Pink', null, false, 'in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (12, 'Goodwell', 'agoodwellb@tamu.edu', 'Female', 'Fuscia', null, true, 'volutpat convallis morbi odio odio elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (13, 'Mayger', 'mmaygerc@bbc.co.uk', 'Male', 'Indigo', null, true, 'donec ut dolor morbi vel lectus in quam fringilla rhoncus mauris enim leo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (14, 'McKeurton', 'gmckeurtond@noaa.gov', 'Male', 'Mauv', null, true, 'ligula in lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (15, 'Drawmer', 'mdrawmere@paypal.com', 'Male', 'Goldenrod', null, false, 'ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu adipiscing'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (16, 'Tessier', 'utessierf@godaddy.com', 'Female', 'Turquoise', null, true, 'ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (17, 'Wardrope', 'jwardropeg@1und1.de', 'Female', 'Fuscia', null, true, 'curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (18, 'Castellaccio', 'rcastellaccioh@narod.ru', 'Female', 'Blue', null, false, 'dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (19, 'Glazier', 'kglazieri@jugem.jp', 'Female', 'Puce', null, false, 'tristique est et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (20, 'McLennan', 'cmclennanj@vk.com', 'Male', 'Orange', null, true, 'leo maecenas pulvinar lobortis est phasellus sit amet erat nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (21, 'Salman', 'ssalmank@youtube.com', 'Male', 'Purple', null, true, 'etiam vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (22, 'Cutress', 'wcutressl@fotki.com', 'Male', 'Turquoise', null, false, 'consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (23, 'Cornes', 'bcornesm@biblegateway.com', 'Female', 'Pink', null, false, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (24, 'Sylvaine', 'psylvainen@aol.com', 'Female', 'Maroon', null, false, 'neque duis bibendum morbi non quam nec dui luctus rutrum nulla tellus in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (25, 'Crolla', 'ccrollao@altervista.org', 'Female', 'Aquamarine', null, false, 'erat fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (26, 'Miebes', 'tmiebesp@php.net', 'Male', 'Goldenrod', null, false, 'primis in faucibus orci luctus et ultrices posuere cubilia curae'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (27, 'Bousquet', 'tbousquetq@over-blog.com', 'Male', 'Purple', null, false, 'pellentesque at nulla suspendisse potenti cras in purus eu magna vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (28, 'Gullberg', 'ogullbergr@bloomberg.com', 'Male', 'Turquoise', null, false, 'id turpis integer aliquet massa id lobortis convallis tortor risus dapibus augue vel accumsan tellus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (29, 'Shackleton', 'nshackletons@friendfeed.com', 'Female', 'Fuscia', null, false, 'sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (30, 'Grasha', 'ograshat@mit.edu', 'Female', 'Maroon', null, false, 'pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (31, 'Elvidge', 'telvidgeu@reuters.com', 'Female', 'Mauv', null, true, 'habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (32, 'Whitham', 'rwhithamv@cafepress.com', 'Female', 'Orange', null, false, 'nulla nunc purus phasellus in felis donec semper sapien a libero nam dui proin leo odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (33, 'Bulloch', 'vbullochw@engadget.com', 'Male', 'Purple', null, true, 'aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (34, 'Lawrinson', 'slawrinsonx@hubpages.com', 'Female', 'Mauv', null, true, 'nunc proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (35, 'Thorn', 'cthorny@friendfeed.com', 'Male', 'Green', null, false, 'lorem id ligula suspendisse ornare consequat lectus in est risus auctor sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (36, 'Yurinov', 'byurinovz@senate.gov', 'Male', 'Mauv', null, false, 'odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (37, 'Arstall', 'garstall10@ftc.gov', 'Male', 'Blue', null, true, 'et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (38, 'Jollye', 'ajollye11@soup.io', 'Female', 'Maroon', null, true, 'suspendisse potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (39, 'Teffrey', 'bteffrey12@ovh.net', 'Female', 'Khaki', null, true, 'magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (40, 'Christopher', 'hchristopher13@theglobeandmail.com', 'Female', 'Turquoise', null, true, 'sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent blandit lacinia erat vestibulum sed magna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (41, 'Eakin', 'jeakin14@constantcontact.com', 'Female', 'Green', null, false, 'ultrices aliquet maecenas leo odio condimentum id luctus nec molestie sed justo pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (42, 'Linzee', 'klinzee15@skyrock.com', 'Male', 'Turquoise', null, true, 'proin interdum mauris non ligula pellentesque ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (43, 'Grigoli', 'ggrigoli16@tinypic.com', 'Male', 'Yellow', null, true, 'pede justo eu massa donec dapibus duis at velit eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (44, 'Gepp', 'ugepp17@intel.com', 'Female', 'Blue', null, false, 'vivamus in felis eu sapien cursus vestibulum proin eu mi nulla ac enim in tempor turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (45, 'Sherwood', 'tsherwood18@indiegogo.com', 'Male', 'Teal', null, false, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (46, 'Gaydon', 'mgaydon19@who.int', 'Male', 'Purple', null, false, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (47, 'Episcopi', 'cepiscopi1a@arstechnica.com', 'Female', 'Crimson', null, false, 'porttitor lacus at turpis donec posuere metus vitae ipsum aliquam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (48, 'Sparshutt', 'psparshutt1b@devhub.com', 'Female', 'Turquoise', null, false, 'congue risus semper porta volutpat quam pede lobortis ligula sit amet eleifend pede libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (49, 'Readhead', 'mreadhead1c@berkeley.edu', 'Male', 'Orange', null, true, 'ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut mauris eget massa tempor convallis nulla neque libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (50, 'Greest', 'rgreest1d@dion.ne.jp', 'Male', 'Purple', null, true, 'sit amet lobortis sapien sapien non mi integer ac neque duis bibendum morbi non quam nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (51, 'Hagerty', 'jhagerty1e@stumbleupon.com', 'Male', 'Fuscia', null, true, 'odio elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla tempus vivamus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (52, 'Beddow', 'sbeddow1f@infoseek.co.jp', 'Male', 'Turquoise', null, false, 'parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (53, 'Silkstone', 'dsilkstone1g@acquirethisname.com', 'Male', 'Crimson', null, true, 'turpis donec posuere metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet diam in magna bibendum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (54, 'Gaisford', 'dgaisford1h@home.pl', 'Female', 'Red', null, false, 'sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (55, 'Behan', 'sbehan1i@oakley.com', 'Female', 'Red', null, true, 'dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (56, 'Cutchie', 'ocutchie1j@baidu.com', 'Male', 'Pink', null, false, 'justo nec condimentum neque sapien placerat ante nulla justo aliquam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (57, 'Morillas', 'lmorillas1k@cbsnews.com', 'Female', 'Yellow', null, false, 'amet sem fusce consequat nulla nisl nunc nisl duis bibendum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (58, 'Fynan', 'kfynan1l@google.com.au', 'Female', 'Pink', null, false, 'ac est lacinia nisi venenatis tristique fusce congue diam id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (59, 'Jahan', 'rjahan1m@europa.eu', 'Female', 'Green', null, false, 'vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (60, 'Manis', 'hmanis1n@example.com', 'Female', 'Pink', null, true, 'lectus pellentesque at nulla suspendisse potenti cras in purus eu magna vulputate luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (61, 'Brambill', 'obrambill1o@xing.com', 'Male', 'Puce', null, true, 'nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (62, 'Heindrick', 'cheindrick1p@miibeian.gov.cn', 'Male', 'Puce', null, true, 'quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (63, 'Hilling', 'lhilling1q@oakley.com', 'Female', 'Aquamarine', null, false, 'pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (64, 'I''anson', 'sianson1r@wiley.com', 'Male', 'Maroon', null, false, 'suscipit ligula in lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (65, 'Macura', 'jmacura1s@usatoday.com', 'Female', 'Crimson', null, true, 'aenean lectus pellentesque eget nunc donec quis orci eget orci vehicula condimentum curabitur in libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (66, 'Tennet', 'ztennet1t@mit.edu', 'Female', 'Fuscia', null, false, 'suspendisse potenti cras in purus eu magna vulputate luctus cum sociis natoque penatibus et magnis dis parturient montes'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (67, 'Brahams', 'wbrahams1u@barnesandnoble.com', 'Male', 'Puce', null, true, 'justo in blandit ultrices enim lorem ipsum dolor sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (68, 'Taplin', 'rtaplin1v@cocolog-nifty.com', 'Male', 'Turquoise', null, false, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (69, 'Hutchence', 'mhutchence1w@bbc.co.uk', 'Female', 'Indigo', null, true, 'amet turpis elementum ligula vehicula consequat morbi a ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (70, 'Hutchence', 'bhutchence1x@ask.com', 'Male', 'Indigo', null, false, 'lacus morbi quis tortor id nulla ultrices aliquet maecenas leo odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (71, 'Cavee', 'dcavee1y@bigcartel.com', 'Male', 'Blue', null, false, 'nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (72, 'Mattacks', 'kmattacks1z@gmpg.org', 'Female', 'Crimson', null, true, 'id pretium iaculis diam erat fermentum justo nec condimentum neque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (73, 'Ferronel', 'rferronel20@themeforest.net', 'Female', 'Turquoise', null, false, 'mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat morbi a ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (74, 'Lent', 'zlent21@miitbeian.gov.cn', 'Male', 'Maroon', null, false, 'curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (75, 'Crowe', 'hcrowe22@woothemes.com', 'Male', 'Maroon', null, true, 'consequat dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (76, 'Fillan', 'cfillan23@freewebs.com', 'Male', 'Khaki', null, false, 'integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (77, 'Synan', 'esynan24@psu.edu', 'Male', 'Red', null, true, 'vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (78, 'Munning', 'cmunning25@mozilla.org', 'Male', 'Turquoise', null, false, 'gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (79, 'Conibere', 'zconibere26@alibaba.com', 'Female', 'Maroon', null, true, 'amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (80, 'Simacek', 'vsimacek27@usda.gov', 'Male', 'Teal', null, true, 'arcu libero rutrum ac lobortis vel dapibus at diam nam tristique tortor eu pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (81, 'Britch', 'kbritch28@yahoo.co.jp', 'Female', 'Yellow', null, true, 'consequat metus sapien ut nunc vestibulum ante ipsum primis in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (82, 'Argile', 'bargile29@homestead.com', 'Female', 'Orange', null, false, 'morbi non quam nec dui luctus rutrum nulla tellus in sagittis dui vel nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (83, 'Clarkin', 'bclarkin2a@nih.gov', 'Male', 'Red', null, true, 'nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (84, 'Polgreen', 'mpolgreen2b@nationalgeographic.com', 'Female', 'Violet', null, true, 'rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (85, 'Ridde', 'mridde2c@diigo.com', 'Male', 'Maroon', null, true, 'metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (86, 'Ricciardelli', 'lricciardelli2d@si.edu', 'Female', 'Orange', null, false, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (87, 'Sharrock', 'csharrock2e@google.com.br', 'Male', 'Red', null, false, 'tortor quis turpis sed ante vivamus tortor duis mattis egestas metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (88, 'Oakenford', 'roakenford2f@ca.gov', 'Male', 'Turquoise', null, false, 'luctus nec molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (89, 'Bale', 'lbale2g@de.vu', 'Male', 'Mauv', null, false, 'integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (90, 'Gemson', 'cgemson2h@phoca.cz', 'Female', 'Green', null, true, 'neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus et ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (91, 'Vynarde', 'mvynarde2i@about.com', 'Male', 'Khaki', null, true, 'integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (92, 'Saban', 'lsaban2j@nps.gov', 'Female', 'Pink', null, false, 'vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (93, 'Buckie', 'rbuckie2k@msn.com', 'Male', 'Pink', null, false, 'elementum nullam varius nulla facilisi cras non velit nec nisi vulputate nonummy maecenas tincidunt'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (94, 'Gabbatiss', 'egabbatiss2l@loc.gov', 'Female', 'Crimson', null, false, 'hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (95, 'Le Leu', 'nleleu2m@shareasale.com', 'Male', 'Mauv', null, false, 'morbi odio odio elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (96, 'Brend', 'sbrend2n@linkedin.com', 'Female', 'Aquamarine', null, false, 'at nunc commodo placerat praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (97, 'Furlow', 'gfurlow2o@alexa.com', 'Male', 'Yellow', null, false, 'molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (98, 'Hedling', 'dhedling2p@eepurl.com', 'Female', 'Purple', null, false, 'morbi non quam nec dui luctus rutrum nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (99, 'Robertshaw', 'zrobertshaw2q@huffingtonpost.com', 'Female', 'Purple', null, true, 'nisl duis bibendum felis sed interdum venenatis turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (100, 'Mettrick', 'nmettrick2r@nih.gov', 'Male', 'Crimson', null, true, 'justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (101, 'Baxstar', 'fbaxstar2s@illinois.edu', 'Male', 'Blue', null, false, 'id mauris vulputate elementum nullam varius nulla facilisi cras non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (102, 'Lount', 'rlount2t@tmall.com', 'Female', 'Yellow', null, true, 'dapibus dolor vel est donec odio justo sollicitudin ut suscipit a feugiat et eros vestibulum ac est lacinia nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (103, 'Pohlak', 'bpohlak2u@skype.com', 'Female', 'Crimson', null, false, 'quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (104, 'Plan', 'kplan2v@uiuc.edu', 'Female', 'Violet', null, true, 'sed accumsan felis ut at dolor quis odio consequat varius integer ac leo pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (105, 'O''Hanlon', 'eohanlon2w@washingtonpost.com', 'Male', 'Aquamarine', null, true, 'at dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (106, 'Waskett', 'awaskett2x@t-online.de', 'Male', 'Mauv', null, false, 'odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam fringilla rhoncus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (107, 'Tallman', 'ltallman2y@list-manage.com', 'Male', 'Violet', null, false, 'ut odio cras mi pede malesuada in imperdiet et commodo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (108, 'Eastcott', 'peastcott2z@dell.com', 'Male', 'Teal', null, true, 'nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris morbi non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (109, 'Jaine', 'ajaine30@accuweather.com', 'Female', 'Aquamarine', null, true, 'amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (110, 'Kitchin', 'mkitchin31@wikimedia.org', 'Female', 'Indigo', null, true, 'viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (111, 'Jesteco', 'gjesteco32@t.co', 'Male', 'Green', null, true, 'eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (112, 'Marshfield', 'imarshfield33@homestead.com', 'Female', 'Khaki', null, false, 'platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (113, 'Leeson', 'gleeson34@ifeng.com', 'Male', 'Green', null, false, 'porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (114, 'Marusic', 'amarusic35@japanpost.jp', 'Male', 'Yellow', null, false, 'a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (115, 'Filler', 'jfiller36@cbsnews.com', 'Female', 'Crimson', null, false, 'blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (116, 'Bezarra', 'nbezarra37@mac.com', 'Female', 'Indigo', null, true, 'penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (117, 'Jude', 'ajude38@google.ca', 'Male', 'Indigo', null, false, 'justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (118, 'Arnoud', 'aarnoud39@ow.ly', 'Male', 'Purple', null, true, 'iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (119, 'Tween', 'ftween3a@squarespace.com', 'Male', 'Turquoise', null, false, 'aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (120, 'Durston', 'pdurston3b@123-reg.co.uk', 'Female', 'Khaki', null, true, 'ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (121, 'Fatscher', 'wfatscher3c@slideshare.net', 'Male', 'Red', null, true, 'consequat lectus in est risus auctor sed tristique in tempus sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (122, 'Sulman', 'asulman3d@businessinsider.com', 'Male', 'Maroon', null, true, 'turpis nec euismod scelerisque quam turpis adipiscing lorem vitae mattis nibh ligula nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (123, 'Creigan', 'acreigan3e@virginia.edu', 'Female', 'Orange', null, true, 'eleifend luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (124, 'Fortnon', 'dfortnon3f@sogou.com', 'Female', 'Goldenrod', null, true, 'maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (125, 'Doerrling', 'bdoerrling3g@unesco.org', 'Female', 'Khaki', null, true, 'diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (126, 'Shenfish', 'ishenfish3h@engadget.com', 'Female', 'Goldenrod', null, true, 'nisi venenatis tristique fusce congue diam id ornare imperdiet sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (127, 'Pigden', 'fpigden3i@joomla.org', 'Male', 'Teal', null, true, 'dui luctus rutrum nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (128, 'Cavendish', 'bcavendish3j@live.com', 'Male', 'Red', null, true, 'in lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (129, 'Mitkin', 'wmitkin3k@gov.uk', 'Female', 'Red', null, true, 'cursus vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (130, 'Weavers', 'lweavers3l@unc.edu', 'Male', 'Indigo', null, true, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (131, 'Fitzroy', 'sfitzroy3m@timesonline.co.uk', 'Female', 'Puce', null, true, 'auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo morbi ut odio cras mi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (132, 'Voas', 'kvoas3n@drupal.org', 'Male', 'Goldenrod', null, false, 'arcu libero rutrum ac lobortis vel dapibus at diam nam tristique tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (133, 'Chisholm', 'mchisholm3o@xing.com', 'Female', 'Orange', null, true, 'semper sapien a libero nam dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (134, 'Sullens', 'ksullens3p@latimes.com', 'Male', 'Fuscia', null, true, 'libero nullam sit amet turpis elementum ligula vehicula consequat morbi a ipsum integer a nibh in quis justo maecenas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (135, 'Sollime', 'tsollime3q@dailymotion.com', 'Female', 'Maroon', null, true, 'ut mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (136, 'Bracknell', 'fbracknell3r@lycos.com', 'Female', 'Mauv', null, false, 'adipiscing elit proin risus praesent lectus vestibulum quam sapien varius ut blandit non interdum in ante vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (137, 'Saltmarshe', 'asaltmarshe3s@digg.com', 'Male', 'Turquoise', null, false, 'cras pellentesque volutpat dui maecenas tristique est et tempus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (138, 'Conaboy', 'oconaboy3t@un.org', 'Male', 'Aquamarine', null, false, 'luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (139, 'Betteney', 'ibetteney3u@opensource.org', 'Female', 'Aquamarine', null, false, 'lobortis ligula sit amet eleifend pede libero quis orci nullam molestie nibh in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (140, 'Jezzard', 'ojezzard3v@livejournal.com', 'Female', 'Yellow', null, false, 'donec ut dolor morbi vel lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (141, 'Starte', 'astarte3w@zdnet.com', 'Female', 'Aquamarine', null, true, 'adipiscing elit proin risus praesent lectus vestibulum quam sapien varius ut blandit non interdum in ante vestibulum ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (142, 'Gladtbach', 'tgladtbach3x@arstechnica.com', 'Female', 'Goldenrod', null, true, 'adipiscing elit proin interdum mauris non ligula pellentesque ultrices phasellus id sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (143, 'Harbage', 'hharbage3y@xing.com', 'Female', 'Maroon', null, true, 'ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (144, 'Matussov', 'hmatussov3z@hugedomains.com', 'Female', 'Puce', null, true, 'sollicitudin mi sit amet lobortis sapien sapien non mi integer ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (145, 'Yellowlee', 'ryellowlee40@ft.com', 'Male', 'Aquamarine', null, true, 'nunc nisl duis bibendum felis sed interdum venenatis turpis enim blandit mi in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (146, 'Kildahl', 'ykildahl41@dailymotion.com', 'Male', 'Turquoise', null, false, 'sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (147, 'Cattach', 'mcattach42@marriott.com', 'Male', 'Blue', null, true, 'justo morbi ut odio cras mi pede malesuada in imperdiet et commodo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (148, 'Skeldon', 'sskeldon43@reference.com', 'Female', 'Maroon', null, false, 'penatibus et magnis dis parturient montes nascetur ridiculus mus etiam vel augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (149, 'Siggee', 'bsiggee44@ted.com', 'Male', 'Aquamarine', null, true, 'sed magna at nunc commodo placerat praesent blandit nam nulla integer pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (150, 'Kelner', 'rkelner45@parallels.com', 'Male', 'Purple', null, true, 'curae donec pharetra magna vestibulum aliquet ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (151, 'Whittlesea', 'jwhittlesea46@comsenz.com', 'Male', 'Orange', null, false, 'enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (152, 'Khalid', 'tkhalid47@diigo.com', 'Male', 'Puce', null, false, 'curae nulla dapibus dolor vel est donec odio justo sollicitudin ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (153, 'Maypowder', 'dmaypowder48@webnode.com', 'Male', 'Orange', null, false, 'ut nunc vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam vitae quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (154, 'Lineen', 'blineen49@upenn.edu', 'Male', 'Orange', null, false, 'dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (155, 'Jennick', 'bjennick4a@goo.gl', 'Female', 'Goldenrod', null, true, 'velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (156, 'Shone', 'hshone4b@godaddy.com', 'Male', 'Maroon', null, false, 'nullam varius nulla facilisi cras non velit nec nisi vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (157, 'Hallgath', 'challgath4c@cloudflare.com', 'Female', 'Teal', null, true, 'primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (158, 'Walcar', 'cwalcar4d@etsy.com', 'Male', 'Goldenrod', null, false, 'sem duis aliquam convallis nunc proin at turpis a pede posuere nonummy'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (159, 'McCray', 'nmccray4e@hhs.gov', 'Female', 'Indigo', null, true, 'placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (160, 'Siddaley', 'wsiddaley4f@hexun.com', 'Male', 'Orange', null, true, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (161, 'Fitzhenry', 'efitzhenry4g@upenn.edu', 'Female', 'Khaki', null, true, 'risus semper porta volutpat quam pede lobortis ligula sit amet eleifend pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (162, 'Worboy', 'vworboy4h@weather.com', 'Male', 'Puce', null, false, 'vel enim sit amet nunc viverra dapibus nulla suscipit ligula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (163, 'O''Cleary', 'eocleary4i@guardian.co.uk', 'Male', 'Khaki', null, true, 'rhoncus dui vel sem sed sagittis nam congue risus semper porta'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (164, 'Vyel', 'mvyel4j@ehow.com', 'Male', 'Yellow', null, true, 'dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (165, 'O'' Connell', 'goconnell4k@wix.com', 'Male', 'Puce', null, true, 'porttitor lorem id ligula suspendisse ornare consequat lectus in est risus auctor sed tristique in tempus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (166, 'Johnke', 'rjohnke4l@elegantthemes.com', 'Female', 'Pink', null, false, 'nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat morbi a ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (167, 'Martinez', 'bmartinez4m@whitehouse.gov', 'Female', 'Mauv', null, true, 'magna ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (168, 'Paggitt', 'ppaggitt4n@cdbaby.com', 'Male', 'Khaki', null, true, 'lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu sapien cursus vestibulum proin eu mi nulla ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (169, 'Riceards', 'friceards4o@so-net.ne.jp', 'Male', 'Puce', null, false, 'sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam congue risus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (170, 'Pedroni', 'wpedroni4p@oakley.com', 'Male', 'Fuscia', null, true, 'ipsum praesent blandit lacinia erat vestibulum sed magna at nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (171, 'Aitcheson', 'maitcheson4q@globo.com', 'Male', 'Purple', null, false, 'mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel enim sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (172, 'Churly', 'echurly4r@parallels.com', 'Female', 'Green', null, true, 'accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (173, 'Matyukon', 'tmatyukon4s@bandcamp.com', 'Female', 'Yellow', null, false, 'nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (174, 'Chalk', 'mchalk4t@google.it', 'Female', 'Khaki', null, false, 'luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (175, 'Savary', 'isavary4u@sphinn.com', 'Female', 'Khaki', null, false, 'praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (176, 'Maleck', 'bmaleck4v@xrea.com', 'Female', 'Pink', null, false, 'rhoncus dui vel sem sed sagittis nam congue risus semper porta volutpat quam pede lobortis ligula sit amet eleifend pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (177, 'Pawfoot', 'mpawfoot4w@tumblr.com', 'Male', 'Indigo', null, false, 'malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (178, 'Benting', 'tbenting4x@1688.com', 'Male', 'Aquamarine', null, false, 'aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (179, 'McPeake', 'lmcpeake4y@networksolutions.com', 'Female', 'Green', null, false, 'suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (180, 'Vanlint', 'cvanlint4z@php.net', 'Female', 'Yellow', null, true, 'aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (181, 'Velde', 'avelde50@flickr.com', 'Female', 'Purple', null, false, 'justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (182, 'Colliard', 'tcolliard51@ebay.com', 'Male', 'Pink', null, false, 'aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (183, 'Tidman', 'atidman52@comcast.net', 'Female', 'Violet', null, false, 'at velit eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (184, 'Sadgrove', 'msadgrove53@eepurl.com', 'Female', 'Maroon', null, false, 'in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (185, 'Matzen', 'dmatzen54@upenn.edu', 'Female', 'Crimson', null, false, 'vehicula condimentum curabitur in libero ut massa volutpat convallis morbi odio odio elementum eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (186, 'Byforth', 'fbyforth55@alexa.com', 'Female', 'Green', null, false, 'aliquam non mauris morbi non lectus aliquam sit amet diam in magna bibendum imperdiet nullam orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (187, 'Bromell', 'hbromell56@dmoz.org', 'Male', 'Maroon', null, true, 'sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut mauris eget massa'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (188, 'Jemison', 'jjemison57@ebay.com', 'Male', 'Turquoise', null, true, 'sit amet nulla quisque arcu libero rutrum ac lobortis vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (189, 'Blezard', 'gblezard58@ox.ac.uk', 'Male', 'Aquamarine', null, true, 'at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (190, 'Willock', 'awillock59@people.com.cn', 'Female', 'Teal', null, false, 'augue aliquam erat volutpat in congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (191, 'Leadbeater', 'dleadbeater5a@seesaa.net', 'Female', 'Green', null, true, 'aliquam sit amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (192, 'Keymar', 'ekeymar5b@blogspot.com', 'Male', 'Aquamarine', null, true, 'odio elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (193, 'Hazelby', 'dhazelby5c@live.com', 'Female', 'Violet', null, true, 'ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem vitae mattis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (194, 'Teggin', 'eteggin5d@amazon.co.uk', 'Male', 'Red', null, true, 'nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (195, 'Kinsell', 'skinsell5e@hp.com', 'Male', 'Mauv', null, true, 'pede libero quis orci nullam molestie nibh in lectus pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (196, 'Daniels', 'hdaniels5f@webeden.co.uk', 'Male', 'Aquamarine', null, true, 'nulla justo aliquam quis turpis eget elit sodales scelerisque mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (197, 'Abramov', 'cabramov5g@cnbc.com', 'Male', 'Indigo', null, true, 'at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (198, 'Francey', 'tfrancey5h@gov.uk', 'Male', 'Indigo', null, false, 'sodales sed tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (199, 'Pieroni', 'tpieroni5i@g.co', 'Male', 'Fuscia', null, true, 'elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (200, 'Penhaligon', 'rpenhaligon5j@va.gov', 'Female', 'Violet', null, true, 'gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (201, 'Pawlowicz', 'kpawlowicz5k@uol.com.br', 'Female', 'Aquamarine', null, true, 'mauris viverra diam vitae quam suspendisse potenti nullam porttitor lacus at turpis donec posuere metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (202, 'Marcombe', 'dmarcombe5l@time.com', 'Female', 'Red', null, false, 'molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget nunc donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (203, 'Joannic', 'bjoannic5m@wiley.com', 'Female', 'Green', null, false, 'in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (204, 'Corballis', 'lcorballis5n@shop-pro.jp', 'Male', 'Blue', null, false, 'sollicitudin ut suscipit a feugiat et eros vestibulum ac est lacinia nisi venenatis tristique fusce'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (205, 'Jacquot', 'ajacquot5o@github.io', 'Female', 'Crimson', null, true, 'libero rutrum ac lobortis vel dapibus at diam nam tristique tortor eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (206, 'Harnwell', 'vharnwell5p@ustream.tv', 'Male', 'Indigo', null, true, 'scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (207, 'Ridulfo', 'kridulfo5q@instagram.com', 'Female', 'Turquoise', null, false, 'eleifend quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (208, 'Pietrzyk', 'lpietrzyk5r@hp.com', 'Female', 'Mauv', null, false, 'non ligula pellentesque ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu adipiscing'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (209, 'Lapsley', 'dlapsley5s@addthis.com', 'Male', 'Indigo', null, true, 'odio porttitor id consequat in consequat ut nulla sed accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (210, 'Cawsby', 'gcawsby5t@liveinternet.ru', 'Male', 'Maroon', null, true, 'risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia sapien quis libero nullam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (211, 'Bedrosian', 'sbedrosian5u@google.ca', 'Male', 'Puce', null, true, 'amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (212, 'Fidian', 'bfidian5v@ted.com', 'Male', 'Mauv', null, true, 'consequat ut nulla sed accumsan felis ut at dolor quis odio consequat varius integer ac leo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (213, 'Roby', 'croby5w@joomla.org', 'Male', 'Green', null, true, 'semper interdum mauris ullamcorper purus sit amet nulla quisque arcu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (214, 'Baudichon', 'abaudichon5x@acquirethisname.com', 'Male', 'Red', null, false, 'elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (215, 'Hurlestone', 'mhurlestone5y@wired.com', 'Male', 'Crimson', null, false, 'ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (216, 'Squeers', 'tsqueers5z@nih.gov', 'Female', 'Red', null, true, 'elementum pellentesque quisque porta volutpat erat quisque erat eros viverra eget congue eget semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (217, 'Rourke', 'arourke60@amazon.co.jp', 'Male', 'Yellow', null, false, 'sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (218, 'Jandourek', 'ijandourek61@virginia.edu', 'Female', 'Green', null, true, 'volutpat in congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (219, 'Curlis', 'jcurlis62@hatena.ne.jp', 'Male', 'Goldenrod', null, false, 'luctus nec molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (220, 'Salvage', 'msalvage63@xinhuanet.com', 'Male', 'Puce', null, false, 'ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (221, 'Clubbe', 'bclubbe64@soup.io', 'Male', 'Red', null, true, 'potenti nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris morbi non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (222, 'Kingsnorth', 'dkingsnorth65@opera.com', 'Female', 'Aquamarine', null, false, 'integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (223, 'Crame', 'ncrame66@skype.com', 'Female', 'Goldenrod', null, true, 'ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (224, 'Waith', 'dwaith67@dell.com', 'Female', 'Purple', null, true, 'tellus semper interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum ac lobortis vel dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (225, 'Duigenan', 'dduigenan68@shareasale.com', 'Male', 'Khaki', null, true, 'at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (226, 'Crisford', 'lcrisford69@rakuten.co.jp', 'Female', 'Crimson', null, true, 'duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (227, 'Grevile', 'zgrevile6a@epa.gov', 'Male', 'Blue', null, false, 'justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (228, 'Cotty', 'ecotty6b@wordpress.com', 'Male', 'Fuscia', null, false, 'porta volutpat quam pede lobortis ligula sit amet eleifend pede libero quis orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (229, 'Brierly', 'jbrierly6c@skype.com', 'Male', 'Red', null, false, 'eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas metus aenean'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (230, 'McDonnell', 'tmcdonnell6d@soup.io', 'Male', 'Violet', null, false, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (231, 'Klemps', 'lklemps6e@eventbrite.com', 'Female', 'Red', null, true, 'enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (232, 'Smout', 'bsmout6f@cyberchimps.com', 'Female', 'Orange', null, false, 'id consequat in consequat ut nulla sed accumsan felis ut at dolor quis odio consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (233, 'Sagar', 'jsagar6g@independent.co.uk', 'Male', 'Indigo', null, true, 'at vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget orci vehicula condimentum curabitur in libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (234, 'Corstorphine', 'rcorstorphine6h@dailymotion.com', 'Male', 'Goldenrod', null, true, 'nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (235, 'Barkess', 'jbarkess6i@webs.com', 'Female', 'Maroon', null, false, 'in ante vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae duis faucibus accumsan odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (236, 'Tole', 'ptole6j@rakuten.co.jp', 'Male', 'Red', null, false, 'elementum nullam varius nulla facilisi cras non velit nec nisi vulputate nonummy maecenas tincidunt lacus at velit vivamus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (237, 'Gebhardt', 'zgebhardt6k@patch.com', 'Female', 'Goldenrod', null, true, 'justo morbi ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (238, 'Beetham', 'abeetham6l@npr.org', 'Female', 'Pink', null, false, 'odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (239, 'Ravel', 'yravel6m@histats.com', 'Male', 'Red', null, false, 'ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (240, 'Skepper', 'mskepper6n@harvard.edu', 'Female', 'Turquoise', null, false, 'sed vestibulum sit amet cursus id turpis integer aliquet massa'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (241, 'Glanders', 'tglanders6o@newsvine.com', 'Female', 'Pink', null, true, 'metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (242, 'Penny', 'rpenny6p@ihg.com', 'Male', 'Violet', null, true, 'massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (243, 'Bendley', 'gbendley6q@discuz.net', 'Female', 'Fuscia', null, true, 'odio donec vitae nisi nam ultrices libero non mattis pulvinar'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (244, 'Ruben', 'mruben6r@w3.org', 'Male', 'Indigo', null, true, 'diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere felis sed lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (245, 'Rammell', 'mrammell6s@yale.edu', 'Female', 'Red', null, false, 'integer ac neque duis bibendum morbi non quam nec dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (246, 'Carcass', 'scarcass6t@reuters.com', 'Male', 'Yellow', null, true, 'sed interdum venenatis turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (247, 'Tompkin', 'btompkin6u@dailymail.co.uk', 'Female', 'Crimson', null, false, 'luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (248, 'Hencke', 'nhencke6v@eepurl.com', 'Female', 'Red', null, true, 'libero ut massa volutpat convallis morbi odio odio elementum eu interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (249, 'Bertolin', 'nbertolin6w@odnoklassniki.ru', 'Female', 'Khaki', null, false, 'ullamcorper augue a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (250, 'L'' Anglois', 'alanglois6x@qq.com', 'Male', 'Aquamarine', null, true, 'purus phasellus in felis donec semper sapien a libero nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (251, 'Smaleman', 'hsmaleman6y@free.fr', 'Male', 'Purple', null, false, 'in libero ut massa volutpat convallis morbi odio odio elementum eu interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (252, 'Malan', 'rmalan6z@smugmug.com', 'Male', 'Mauv', null, false, 'posuere cubilia curae mauris viverra diam vitae quam suspendisse potenti nullam porttitor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (253, 'Iacobassi', 'wiacobassi70@ft.com', 'Male', 'Maroon', null, false, 'eu interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (254, 'Weymont', 'bweymont71@paypal.com', 'Female', 'Orange', null, false, 'ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (255, 'Alliot', 'galliot72@gizmodo.com', 'Female', 'Puce', null, false, 'luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (256, 'Berrey', 'oberrey73@angelfire.com', 'Male', 'Indigo', null, true, 'pharetra magna ac consequat metus sapien ut nunc vestibulum ante ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (257, 'Wolver', 'swolver74@mac.com', 'Male', 'Fuscia', null, false, 'proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (258, 'Trotman', 'gtrotman75@weebly.com', 'Male', 'Blue', null, false, 'sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (259, 'Iliffe', 'iiliffe76@gov.uk', 'Male', 'Goldenrod', null, false, 'turpis adipiscing lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (260, 'Ungaretti', 'bungaretti77@addthis.com', 'Male', 'Fuscia', null, false, 'ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (261, 'Lettice', 'elettice78@vistaprint.com', 'Female', 'Aquamarine', null, true, 'sapien cursus vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (262, 'Ricard', 'rricard79@yahoo.com', 'Female', 'Pink', null, true, 'tristique in tempus sit amet sem fusce consequat nulla nisl nunc nisl duis bibendum felis sed interdum venenatis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (263, 'Omand', 'momand7a@netvibes.com', 'Female', 'Khaki', null, false, 'sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus augue vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (264, 'Sorrie', 'bsorrie7b@dropbox.com', 'Female', 'Maroon', null, false, 'ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit a feugiat et eros'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (265, 'Breawood', 'gbreawood7c@facebook.com', 'Female', 'Fuscia', null, true, 'amet lobortis sapien sapien non mi integer ac neque duis bibendum morbi non quam nec dui luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (266, 'Haddon', 'bhaddon7d@wix.com', 'Female', 'Teal', null, true, 'vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (267, 'Kell', 'wkell7e@biglobe.ne.jp', 'Male', 'Blue', null, true, 'amet lobortis sapien sapien non mi integer ac neque duis bibendum morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (268, 'Stidever', 'cstidever7f@deviantart.com', 'Female', 'Puce', null, false, 'duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (269, 'Abbots', 'aabbots7g@webnode.com', 'Female', 'Fuscia', null, false, 'maecenas tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque quisque porta'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (270, 'Garnam', 'agarnam7h@deliciousdays.com', 'Male', 'Mauv', null, true, 'maecenas leo odio condimentum id luctus nec molestie sed justo pellentesque viverra pede ac diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (271, 'Whitesel', 'cwhitesel7i@mail.ru', 'Male', 'Fuscia', null, false, 'turpis donec posuere metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet diam in magna bibendum imperdiet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (272, 'Bluck', 'sbluck7j@whitehouse.gov', 'Male', 'Purple', null, false, 'nam nulla integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse ornare'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (273, 'Eymer', 'neymer7k@unicef.org', 'Male', 'Violet', null, true, 'gravida nisi at nibh in hac habitasse platea dictumst aliquam augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (274, 'Upston', 'cupston7l@wikia.com', 'Male', 'Blue', null, false, 'maecenas tristique est et tempus semper est quam pharetra magna ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (275, 'Jose', 'kjose7m@cornell.edu', 'Male', 'Khaki', null, true, 'lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (276, 'Pellamont', 'spellamont7n@moonfruit.com', 'Female', 'Goldenrod', null, false, 'amet justo morbi ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (277, 'Cremen', 'rcremen7o@google.de', 'Male', 'Indigo', null, true, 'nullam sit amet turpis elementum ligula vehicula consequat morbi a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (278, 'Drivers', 'mdrivers7p@instagram.com', 'Male', 'Red', null, true, 'orci luctus et ultrices posuere cubilia curae mauris viverra diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (279, 'Climar', 'nclimar7q@baidu.com', 'Female', 'Red', null, true, 'eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (280, 'Murfett', 'hmurfett7r@upenn.edu', 'Male', 'Teal', null, false, 'turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (281, 'Pearcy', 'dpearcy7s@naver.com', 'Female', 'Aquamarine', null, false, 'cras non velit nec nisi vulputate nonummy maecenas tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (282, 'Arkil', 'darkil7t@businesswire.com', 'Male', 'Yellow', null, true, 'id ornare imperdiet sapien urna pretium nisl ut volutpat sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (283, 'Mouton', 'cmouton7u@twitter.com', 'Female', 'Maroon', null, false, 'faucibus cursus urna ut tellus nulla ut erat id mauris vulputate elementum nullam varius nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (284, 'Bastistini', 'wbastistini7v@sakura.ne.jp', 'Male', 'Red', null, true, 'a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (285, 'Sanches', 'rsanches7w@slideshare.net', 'Female', 'Khaki', null, false, 'consequat lectus in est risus auctor sed tristique in tempus sit amet sem fusce consequat nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (286, 'Merle', 'bmerle7x@theglobeandmail.com', 'Female', 'Pink', null, false, 'porttitor lorem id ligula suspendisse ornare consequat lectus in est risus auctor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (287, 'Gerrard', 'egerrard7y@columbia.edu', 'Male', 'Aquamarine', null, false, 'vulputate nonummy maecenas tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque quisque porta volutpat erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (288, 'MacKereth', 'tmackereth7z@microsoft.com', 'Female', 'Fuscia', null, true, 'libero ut massa volutpat convallis morbi odio odio elementum eu interdum eu tincidunt in leo maecenas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (289, 'Thirtle', 'kthirtle80@businesswire.com', 'Male', 'Green', null, true, 'dolor sit amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque ultrices phasellus id sapien in sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (290, 'Greenlees', 'kgreenlees81@creativecommons.org', 'Female', 'Maroon', null, false, 'cursus urna ut tellus nulla ut erat id mauris vulputate elementum nullam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (291, 'Aaronson', 'naaronson82@umn.edu', 'Female', 'Fuscia', null, false, 'ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam vitae quam suspendisse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (292, 'Lambarton', 'flambarton83@si.edu', 'Male', 'Violet', null, true, 'sed nisl nunc rhoncus dui vel sem sed sagittis nam congue risus semper porta volutpat quam pede lobortis ligula sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (293, 'Mabbott', 'amabbott84@storify.com', 'Female', 'Indigo', null, false, 'duis mattis egestas metus aenean fermentum donec ut mauris eget massa tempor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (294, 'Gosart', 'hgosart85@yale.edu', 'Male', 'Goldenrod', null, false, 'adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget orci vehicula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (295, 'Scini', 'sscini86@ebay.co.uk', 'Female', 'Puce', null, true, 'vivamus in felis eu sapien cursus vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (296, 'Nare', 'knare87@unicef.org', 'Male', 'Blue', null, false, 'fusce congue diam id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu sed augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (297, 'Doughty', 'kdoughty88@seattletimes.com', 'Male', 'Indigo', null, true, 'curabitur gravida nisi at nibh in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (298, 'Hammant', 'thammant89@mlb.com', 'Male', 'Teal', null, true, 'tellus nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (299, 'Howsin', 'whowsin8a@i2i.jp', 'Male', 'Khaki', null, false, 'at velit vivamus vel nulla eget eros elementum pellentesque quisque porta'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (300, 'Hunt', 'dhunt8b@apple.com', 'Male', 'Pink', null, true, 'libero nam dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (301, 'Cattell', 'acattell8c@discuz.net', 'Female', 'Orange', null, false, 'amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (302, 'todor', 'dtodor8d@independent.co.uk', 'Female', 'Puce', null, true, 'justo sollicitudin ut suscipit a feugiat et eros vestibulum ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (303, 'McDuffie', 'vmcduffie8e@berkeley.edu', 'Female', 'Teal', null, false, 'vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (304, 'Fitzpatrick', 'sfitzpatrick8f@java.com', 'Male', 'Red', null, true, 'at velit eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (305, 'McNair', 'hmcnair8g@youtube.com', 'Male', 'Violet', null, true, 'suspendisse potenti in eleifend quam a odio in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (306, 'Vasenkov', 'lvasenkov8h@istockphoto.com', 'Male', 'Yellow', null, true, 'rutrum at lorem integer tincidunt ante vel ipsum praesent blandit lacinia erat vestibulum sed magna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (307, 'Folcarelli', 'nfolcarelli8i@bravesites.com', 'Male', 'Pink', null, false, 'aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (308, 'Cursey', 'dcursey8j@java.com', 'Male', 'Fuscia', null, false, 'rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (309, 'Keneleyside', 'zkeneleyside8k@google.it', 'Male', 'Khaki', null, false, 'in congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (310, 'Ellaway', 'sellaway8l@skype.com', 'Male', 'Crimson', null, true, 'odio curabitur convallis duis consequat dui nec nisi volutpat eleifend'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (311, 'Jervois', 'ejervois8m@sciencedirect.com', 'Male', 'Turquoise', null, false, 'in porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (312, 'Danaher', 'rdanaher8n@infoseek.co.jp', 'Male', 'Aquamarine', null, false, 'posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (313, 'McConnachie', 'amcconnachie8o@disqus.com', 'Female', 'Fuscia', null, false, 'mi nulla ac enim in tempor turpis nec euismod scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (314, 'Seeney', 'gseeney8p@purevolume.com', 'Male', 'Mauv', null, false, 'ligula nec sem duis aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (315, 'Knapman', 'jknapman8q@soup.io', 'Female', 'Khaki', null, false, 'massa id nisl venenatis lacinia aenean sit amet justo morbi ut odio cras mi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (316, 'Gianelli', 'ngianelli8r@paginegialle.it', 'Female', 'Teal', null, false, 'fermentum donec ut mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (317, 'Garaway', 'ggaraway8s@freewebs.com', 'Male', 'Crimson', null, true, 'pharetra magna ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (318, 'McCarlie', 'mmccarlie8t@php.net', 'Female', 'Khaki', null, false, 'interdum venenatis turpis enim blandit mi in porttitor pede justo eu massa donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (319, 'Zappel', 'hzappel8u@lycos.com', 'Male', 'Teal', null, true, 'etiam vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (320, 'Eggle', 'keggle8v@dyndns.org', 'Male', 'Purple', null, false, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (321, 'Fandrich', 'mfandrich8w@posterous.com', 'Female', 'Orange', null, true, 'condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (322, 'Lundberg', 'slundberg8x@csmonitor.com', 'Male', 'Yellow', null, false, 'sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (323, 'Keymar', 'akeymar8y@yahoo.co.jp', 'Female', 'Aquamarine', null, false, 'dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (324, 'McDonnell', 'gmcdonnell8z@cdc.gov', 'Male', 'Blue', null, false, 'sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in congue etiam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (325, 'Early', 'eearly90@about.com', 'Male', 'Puce', null, true, 'nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (326, 'McAuslan', 'hmcauslan91@multiply.com', 'Female', 'Indigo', null, false, 'luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (327, 'Todman', 'ttodman92@npr.org', 'Male', 'Purple', null, true, 'id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (328, 'Figgen', 'afiggen93@taobao.com', 'Female', 'Yellow', null, false, 'sagittis dui vel nisl duis ac nibh fusce lacus purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (329, 'Robecon', 'zrobecon94@naver.com', 'Male', 'Red', null, false, 'turpis sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (330, 'Nodin', 'cnodin95@devhub.com', 'Female', 'Indigo', null, false, 'nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (331, 'Jacke', 'rjacke96@twitpic.com', 'Male', 'Pink', null, false, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (332, 'Simononsky', 'csimononsky97@bloglovin.com', 'Female', 'Green', null, true, 'integer aliquet massa id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (333, 'Staddart', 'mstaddart98@squidoo.com', 'Male', 'Yellow', null, true, 'etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (334, 'Sprague', 'lsprague99@wunderground.com', 'Male', 'Puce', null, true, 'fusce consequat nulla nisl nunc nisl duis bibendum felis sed interdum venenatis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (335, 'Tootell', 'rtootell9a@chronoengine.com', 'Female', 'Crimson', null, false, 'nisl duis bibendum felis sed interdum venenatis turpis enim blandit mi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (336, 'Hatchett', 'dhatchett9b@bravesites.com', 'Female', 'Teal', null, true, 'enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (337, 'McTrustam', 'bmctrustam9c@senate.gov', 'Female', 'Blue', null, true, 'nunc purus phasellus in felis donec semper sapien a libero nam dui proin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (338, 'Ingry', 'bingry9d@reddit.com', 'Male', 'Maroon', null, true, 'molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (339, 'Fadell', 'lfadell9e@nyu.edu', 'Female', 'Yellow', null, false, 'odio condimentum id luctus nec molestie sed justo pellentesque viverra pede ac diam cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (340, 'Wharram', 'jwharram9f@slate.com', 'Male', 'Green', null, false, 'in sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (341, 'Hasslocher', 'lhasslocher9g@epa.gov', 'Male', 'Aquamarine', null, true, 'in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (342, 'Safhill', 'rsafhill9h@bravesites.com', 'Male', 'Red', null, true, 'est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (343, 'Corcoran', 'rcorcoran9i@seesaa.net', 'Male', 'Khaki', null, true, 'nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (344, 'Hallwell', 'khallwell9j@si.edu', 'Male', 'Maroon', null, false, 'eget eros elementum pellentesque quisque porta volutpat erat quisque erat eros viverra eget congue eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (345, 'Shillum', 'tshillum9k@jimdo.com', 'Female', 'Green', null, true, 'nunc purus phasellus in felis donec semper sapien a libero nam dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (346, 'Whittenbury', 'jwhittenbury9l@tamu.edu', 'Male', 'Green', null, false, 'magna vestibulum aliquet ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (347, 'Irnys', 'cirnys9m@state.gov', 'Male', 'Maroon', null, true, 'congue diam id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (348, 'Brenston', 'cbrenston9n@fc2.com', 'Female', 'Turquoise', null, false, 'platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (349, 'Caldecot', 'rcaldecot9o@springer.com', 'Male', 'Puce', null, false, 'ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (350, 'Robardley', 'erobardley9p@biblegateway.com', 'Female', 'Goldenrod', null, true, 'hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (351, 'MacAleese', 'jmacaleese9q@netvibes.com', 'Female', 'Fuscia', null, true, 'sit amet cursus id turpis integer aliquet massa id lobortis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (352, 'Vials', 'nvials9r@businessinsider.com', 'Male', 'Maroon', null, false, 'quisque erat eros viverra eget congue eget semper rutrum nulla nunc purus phasellus in felis donec semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (353, 'Grundell', 'ggrundell9s@unc.edu', 'Male', 'Orange', null, true, 'id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (354, 'Windsor', 'swindsor9t@goodreads.com', 'Female', 'Maroon', null, true, 'in sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (355, 'Geydon', 'ggeydon9u@devhub.com', 'Male', 'Indigo', null, true, 'enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (356, 'O''Donnelly', 'hodonnelly9v@aboutads.info', 'Male', 'Turquoise', null, true, 'dui vel sem sed sagittis nam congue risus semper porta volutpat quam pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (357, 'Massei', 'dmassei9w@edublogs.org', 'Female', 'Mauv', null, false, 'vel augue vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (358, 'Calendar', 'ucalendar9x@technorati.com', 'Female', 'Pink', null, false, 'hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (359, 'Brigshaw', 'tbrigshaw9y@nba.com', 'Male', 'Turquoise', null, false, 'natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (360, 'Spellessy', 'pspellessy9z@t.co', 'Male', 'Purple', null, false, 'gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (361, 'Yarnley', 'cyarnleya0@nationalgeographic.com', 'Female', 'Puce', null, false, 'blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (362, 'Arundel', 'narundela1@cyberchimps.com', 'Male', 'Crimson', null, true, 'auctor sed tristique in tempus sit amet sem fusce consequat nulla nisl nunc nisl duis bibendum felis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (363, 'Lohde', 'wlohdea2@sohu.com', 'Female', 'Mauv', null, false, 'cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (364, 'Kingescot', 'tkingescota3@a8.net', 'Female', 'Indigo', null, true, 'accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (365, 'Fewings', 'afewingsa4@skype.com', 'Male', 'Orange', null, false, 'lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (366, 'Muzzini', 'fmuzzinia5@creativecommons.org', 'Female', 'Khaki', null, false, 'id luctus nec molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (367, 'Dragonette', 'pdragonettea6@telegraph.co.uk', 'Male', 'Khaki', null, true, 'sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (368, 'Hannibal', 'phannibala7@cbc.ca', 'Male', 'Puce', null, true, 'lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia sapien quis libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (369, 'Schwander', 'kschwandera8@gmpg.org', 'Male', 'Goldenrod', null, false, 'et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (370, 'Basek', 'sbaseka9@ox.ac.uk', 'Female', 'Violet', null, true, 'neque duis bibendum morbi non quam nec dui luctus rutrum nulla tellus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (371, 'MacConnell', 'kmacconnellaa@whitehouse.gov', 'Female', 'Turquoise', null, true, 'eleifend donec ut dolor morbi vel lectus in quam fringilla rhoncus mauris enim'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (372, 'Cragg', 'mcraggab@gmpg.org', 'Female', 'Maroon', null, true, 'sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut mauris eget massa tempor convallis nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (373, 'Tyzack', 'vtyzackac@springer.com', 'Male', 'Pink', null, true, 'semper rutrum nulla nunc purus phasellus in felis donec semper sapien a libero nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (374, 'Sprade', 'jspradead@is.gd', 'Male', 'Aquamarine', null, false, 'pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (375, 'Demkowicz', 'ademkowiczae@nbcnews.com', 'Female', 'Pink', null, false, 'morbi non quam nec dui luctus rutrum nulla tellus in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (376, 'Ferrarotti', 'tferrarottiaf@surveymonkey.com', 'Female', 'Green', null, false, 'eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (377, 'Mardell', 'pmardellag@rakuten.co.jp', 'Female', 'Red', null, false, 'erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (378, 'Pipping', 'tpippingah@t.co', 'Male', 'Fuscia', null, false, 'cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (379, 'Stanex', 'mstanexai@reference.com', 'Female', 'Green', null, true, 'ac est lacinia nisi venenatis tristique fusce congue diam id ornare imperdiet sapien urna pretium'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (380, 'Nani', 'mnaniaj@indiatimes.com', 'Male', 'Red', null, false, 'libero nullam sit amet turpis elementum ligula vehicula consequat morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (381, 'Cancutt', 'acancuttak@ox.ac.uk', 'Male', 'Khaki', null, false, 'neque duis bibendum morbi non quam nec dui luctus rutrum nulla tellus in sagittis dui'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (382, 'Gother', 'rgotheral@toplist.cz', 'Male', 'Puce', null, true, 'metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (383, 'Mongin', 'lmonginam@redcross.org', 'Female', 'Teal', null, true, 'felis eu sapien cursus vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (384, 'Cuddehay', 'tcuddehayan@usa.gov', 'Male', 'Goldenrod', null, false, 'sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (385, 'Birdall', 'wbirdallao@lycos.com', 'Female', 'Aquamarine', null, false, 'potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (386, 'Billborough', 'gbillboroughap@infoseek.co.jp', 'Female', 'Blue', null, true, 'enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (387, 'Berrane', 'aberraneaq@amazon.de', 'Female', 'Purple', null, false, 'duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (388, 'Rowberry', 'srowberryar@salon.com', 'Male', 'Fuscia', null, false, 'rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (389, 'Clampett', 'kclampettas@fc2.com', 'Male', 'Indigo', null, true, 'nec nisi volutpat eleifend donec ut dolor morbi vel lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (390, 'Kleine', 'jkleineat@skyrock.com', 'Male', 'Blue', null, false, 'mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (391, 'Massie', 'emassieau@scribd.com', 'Male', 'Crimson', null, false, 'suscipit ligula in lacus curabitur at ipsum ac tellus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (392, 'Lambert', 'mlambertav@skyrock.com', 'Male', 'Mauv', null, false, 'neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (393, 'Hallet', 'lhalletaw@baidu.com', 'Male', 'Maroon', null, false, 'ante vel ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (394, 'Nuton', 'tnutonax@jalbum.net', 'Female', 'Maroon', null, false, 'justo morbi ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (395, 'Faloon', 'jfaloonay@biblegateway.com', 'Female', 'Red', null, true, 'lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (396, 'Bartlam', 'cbartlamaz@nps.gov', 'Male', 'Yellow', null, true, 'sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (397, 'Lewin', 'mlewinb0@whitehouse.gov', 'Female', 'Puce', null, true, 'molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (398, 'Foulis', 'lfoulisb1@networkadvertising.org', 'Female', 'Pink', null, true, 'penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (399, 'Gannaway', 'mgannawayb2@twitter.com', 'Male', 'Violet', null, true, 'odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (400, 'Kleinstub', 'mkleinstubb3@ustream.tv', 'Male', 'Teal', null, false, 'tortor duis mattis egestas metus aenean fermentum donec ut mauris eget massa tempor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (401, 'Chubb', 'dchubbb4@a8.net', 'Female', 'Pink', null, true, 'pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in congue etiam justo etiam pretium iaculis justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (402, 'Boylund', 'nboylundb5@columbia.edu', 'Female', 'Orange', null, true, 'lobortis ligula sit amet eleifend pede libero quis orci nullam molestie nibh in lectus pellentesque at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (403, 'Musslewhite', 'hmusslewhiteb6@nba.com', 'Female', 'Yellow', null, true, 'justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (404, 'Benediktsson', 'cbenediktssonb7@odnoklassniki.ru', 'Female', 'Yellow', null, true, 'pede libero quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (405, 'Tettley', 'gtettleyb8@gravatar.com', 'Male', 'Purple', null, false, 'duis at velit eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (406, 'Spalton', 'pspaltonb9@state.gov', 'Female', 'Aquamarine', null, true, 'vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (407, 'Gouch', 'jgouchba@apache.org', 'Female', 'Red', null, true, 'habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (408, 'Ygo', 'gygobb@hc360.com', 'Female', 'Orange', null, false, 'faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat tortor sollicitudin mi sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (409, 'Eisenberg', 'eeisenbergbc@admin.ch', 'Male', 'Goldenrod', null, true, 'mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (410, 'Whaites', 'cwhaitesbd@msu.edu', 'Male', 'Yellow', null, false, 'amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (411, 'Scryne', 'pscrynebe@dedecms.com', 'Male', 'Mauv', null, false, 'fusce consequat nulla nisl nunc nisl duis bibendum felis sed interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (412, 'Stother', 'astotherbf@wiley.com', 'Female', 'Puce', null, false, 'integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (413, 'Bazoge', 'lbazogebg@google.pl', 'Male', 'Pink', null, true, 'suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (414, 'French', 'ffrenchbh@google.com', 'Female', 'Khaki', null, false, 'suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (415, 'Girardot', 'bgirardotbi@washington.edu', 'Female', 'Maroon', null, true, 'lorem id ligula suspendisse ornare consequat lectus in est risus auctor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (416, 'Presslee', 'mpressleebj@abc.net.au', 'Female', 'Teal', null, true, 'rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (417, 'Megarry', 'fmegarrybk@addthis.com', 'Female', 'Orange', null, true, 'nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (418, 'Majury', 'dmajurybl@hatena.ne.jp', 'Female', 'Red', null, true, 'consequat nulla nisl nunc nisl duis bibendum felis sed interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (419, 'Brooker', 'cbrookerbm@twitter.com', 'Female', 'Khaki', null, false, 'potenti cras in purus eu magna vulputate luctus cum sociis natoque penatibus et magnis dis parturient'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (420, 'Karpov', 'bkarpovbn@hostgator.com', 'Male', 'Mauv', null, false, 'morbi vel lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (421, 'Kettleson', 'mkettlesonbo@rakuten.co.jp', 'Female', 'Aquamarine', null, true, 'curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (422, 'Lillecrop', 'rlillecropbp@amazon.co.uk', 'Male', 'Green', null, true, 'morbi ut odio cras mi pede malesuada in imperdiet et commodo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (423, 'Keiling', 'nkeilingbq@gmpg.org', 'Female', 'Blue', null, false, 'at velit eu est congue elementum in hac habitasse platea'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (424, 'Leydon', 'hleydonbr@nydailynews.com', 'Female', 'Goldenrod', null, true, 'mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (425, 'Gaven', 'mgavenbs@1und1.de', 'Male', 'Mauv', null, false, 'id nulla ultrices aliquet maecenas leo odio condimentum id luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (426, 'Gonsalvez', 'kgonsalvezbt@w3.org', 'Female', 'Goldenrod', null, true, 'mi in porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (427, 'De Filippi', 'jdefilippibu@cbc.ca', 'Male', 'Orange', null, true, 'sed tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (428, 'Gorghetto', 'hgorghettobv@va.gov', 'Male', 'Fuscia', null, true, 'rutrum nulla tellus in sagittis dui vel nisl duis ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (429, 'Mason', 'gmasonbw@exblog.jp', 'Female', 'Orange', null, true, 'dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (430, 'Rawsen', 'grawsenbx@hubpages.com', 'Female', 'Indigo', null, true, 'risus auctor sed tristique in tempus sit amet sem fusce consequat nulla nisl nunc nisl duis bibendum felis sed interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (431, 'Habberjam', 'mhabberjamby@mapy.cz', 'Female', 'Purple', null, true, 'mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (432, 'Ferrari', 'dferraribz@nymag.com', 'Male', 'Aquamarine', null, false, 'ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (433, 'Pinkard', 'spinkardc0@blogspot.com', 'Female', 'Aquamarine', null, false, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (434, 'Checci', 'dcheccic1@hugedomains.com', 'Male', 'Goldenrod', null, true, 'id nisl venenatis lacinia aenean sit amet justo morbi ut odio cras mi pede malesuada in imperdiet et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (435, 'Tourot', 'ltourotc2@dmoz.org', 'Male', 'Green', null, false, 'vitae quam suspendisse potenti nullam porttitor lacus at turpis donec posuere metus vitae ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (436, 'Sinnie', 'gsinniec3@infoseek.co.jp', 'Female', 'Mauv', null, true, 'tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque quisque porta volutpat erat quisque erat eros'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (437, 'Oldknowe', 'soldknowec4@alexa.com', 'Female', 'Crimson', null, false, 'justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (438, 'Pardal', 'hpardalc5@sourceforge.net', 'Female', 'Mauv', null, true, 'blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (439, 'Resun', 'yresunc6@rambler.ru', 'Female', 'Fuscia', null, false, 'rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (440, 'Lodwig', 'mlodwigc7@sourceforge.net', 'Female', 'Violet', null, false, 'donec semper sapien a libero nam dui proin leo odio porttitor id consequat in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (441, 'Carneck', 'mcarneckc8@google.com.au', 'Female', 'Blue', null, false, 'ut at dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (442, 'Lewzey', 'hlewzeyc9@huffingtonpost.com', 'Male', 'Pink', null, false, 'habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (443, 'Boumphrey', 'bboumphreyca@comcast.net', 'Female', 'Mauv', null, true, 'nisl nunc nisl duis bibendum felis sed interdum venenatis turpis enim blandit mi in porttitor pede justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (444, 'Gerritzen', 'agerritzencb@ted.com', 'Female', 'Khaki', null, true, 'tellus nisi eu orci mauris lacinia sapien quis libero nullam sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (445, 'Ragsdale', 'eragsdalecc@blogs.com', 'Female', 'Violet', null, false, 'faucibus orci luctus et ultrices posuere cubilia curae duis faucibus accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (446, 'Ditchett', 'tditchettcd@lulu.com', 'Male', 'Green', null, true, 'eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (447, 'Andrick', 'eandrickce@livejournal.com', 'Female', 'Yellow', null, false, 'placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (448, 'Robelin', 'brobelincf@reddit.com', 'Female', 'Orange', null, true, 'ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (449, 'Stafford', 'cstaffordcg@qq.com', 'Male', 'Turquoise', null, false, 'cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (450, 'Belhome', 'dbelhomech@nydailynews.com', 'Female', 'Green', null, false, 'id ligula suspendisse ornare consequat lectus in est risus auctor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (451, 'Charopen', 'scharopenci@prlog.org', 'Male', 'Teal', null, true, 'ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer pede justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (452, 'Easterby', 'reasterbycj@ed.gov', 'Female', 'Fuscia', null, true, 'venenatis lacinia aenean sit amet justo morbi ut odio cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (453, 'Brandenburg', 'cbrandenburgck@lulu.com', 'Male', 'Fuscia', null, false, 'ultrices posuere cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (454, 'Searsby', 'hsearsbycl@bloomberg.com', 'Female', 'Red', null, true, 'lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet nulla quisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (455, 'Gercke', 'ggerckecm@deliciousdays.com', 'Female', 'Violet', null, false, 'nisl nunc nisl duis bibendum felis sed interdum venenatis turpis enim blandit mi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (456, 'Comerford', 'acomerfordcn@t-online.de', 'Male', 'Mauv', null, false, 'suspendisse potenti nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (457, 'Sargerson', 'dsargersonco@amazon.de', 'Male', 'Teal', null, false, 'sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent blandit lacinia erat vestibulum sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (458, 'Reddick', 'oreddickcp@mediafire.com', 'Male', 'Khaki', null, false, 'faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam vitae quam suspendisse potenti nullam porttitor lacus at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (459, 'Forrest', 'sforrestcq@statcounter.com', 'Male', 'Fuscia', null, true, 'risus auctor sed tristique in tempus sit amet sem fusce'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (460, 'Weall', 'dweallcr@google.co.uk', 'Female', 'Crimson', null, true, 'posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (461, 'Jumel', 'mjumelcs@vk.com', 'Male', 'Aquamarine', null, false, 'ut mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (462, 'Gwillim', 'cgwillimct@stanford.edu', 'Female', 'Blue', null, false, 'justo eu massa donec dapibus duis at velit eu est congue elementum in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (463, 'Sagrott', 'ksagrottcu@wp.com', 'Male', 'Goldenrod', null, true, 'nonummy integer non velit donec diam neque vestibulum eget vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (464, 'Parmby', 'kparmbycv@ibm.com', 'Male', 'Pink', null, true, 'condimentum id luctus nec molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (465, 'Hiddsley', 'ghiddsleycw@acquirethisname.com', 'Female', 'Mauv', null, true, 'orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (466, 'Awin', 'wawincx@ameblo.jp', 'Female', 'Green', null, false, 'dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (467, 'Gulvin', 'hgulvincy@domainmarket.com', 'Male', 'Turquoise', null, false, 'pellentesque quisque porta volutpat erat quisque erat eros viverra eget congue eget semper rutrum nulla nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (468, 'Brockie', 'cbrockiecz@cmu.edu', 'Female', 'Teal', null, true, 'ac tellus semper interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (469, 'McGlade', 'mmcgladed0@chronoengine.com', 'Male', 'Purple', null, true, 'porta volutpat erat quisque erat eros viverra eget congue eget semper rutrum nulla nunc purus phasellus in felis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (470, 'Utterson', 'ruttersond1@edublogs.org', 'Female', 'Teal', null, true, 'nulla ut erat id mauris vulputate elementum nullam varius nulla facilisi cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (471, 'Luciani', 'alucianid2@umn.edu', 'Female', 'Crimson', null, true, 'eleifend quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (472, 'Provest', 'mprovestd3@imageshack.us', 'Female', 'Goldenrod', null, true, 'vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (473, 'Borchardt', 'mborchardtd4@europa.eu', 'Male', 'Aquamarine', null, false, 'curae nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (474, 'Versey', 'tverseyd5@live.com', 'Female', 'Mauv', null, true, 'donec ut dolor morbi vel lectus in quam fringilla rhoncus mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (475, 'Ornillos', 'mornillosd6@about.me', 'Male', 'Teal', null, false, 'neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo morbi ut odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (476, 'Benson', 'xbensond7@whitehouse.gov', 'Male', 'Indigo', null, true, 'nec euismod scelerisque quam turpis adipiscing lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (477, 'Deniscke', 'edeniscked8@tamu.edu', 'Male', 'Blue', null, false, 'et ultrices posuere cubilia curae mauris viverra diam vitae quam suspendisse potenti'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (478, 'Longden', 'jlongdend9@adobe.com', 'Female', 'Purple', null, true, 'consequat dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam fringilla rhoncus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (479, 'Blackhurst', 'kblackhurstda@dion.ne.jp', 'Female', 'Blue', null, false, 'lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (480, 'Eborall', 'teboralldb@wikispaces.com', 'Male', 'Purple', null, true, 'rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (481, 'O''Hagerty', 'aohagertydc@booking.com', 'Male', 'Violet', null, true, 'posuere metus vitae ipsum aliquam non mauris morbi non lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (482, 'Ridsdale', 'vridsdaledd@time.com', 'Female', 'Turquoise', null, false, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (483, 'McGloin', 'cmcgloinde@hostgator.com', 'Female', 'Turquoise', null, true, 'at nunc commodo placerat praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (484, 'Fley', 'gfleydf@nationalgeographic.com', 'Male', 'Aquamarine', null, true, 'ligula nec sem duis aliquam convallis nunc proin at turpis a pede posuere nonummy integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (485, 'Allibone', 'rallibonedg@howstuffworks.com', 'Female', 'Green', null, true, 'ante vel ipsum praesent blandit lacinia erat vestibulum sed magna at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (486, 'Camplen', 'rcamplendh@g.co', 'Female', 'Green', null, true, 'cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (487, 'Kauschke', 'okauschkedi@nifty.com', 'Male', 'Green', null, false, 'lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (488, 'Mateiko', 'qmateikodj@nsw.gov.au', 'Male', 'Purple', null, false, 'vel ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (489, 'Ollerhad', 'tollerhaddk@instagram.com', 'Female', 'Violet', null, false, 'quam turpis adipiscing lorem vitae mattis nibh ligula nec sem duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (490, 'Lackmann', 'dlackmanndl@clickbank.net', 'Male', 'Green', null, true, 'pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (491, 'Ramsdale', 'jramsdaledm@illinois.edu', 'Male', 'Goldenrod', null, false, 'curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (492, 'Whaley', 'hwhaleydn@state.gov', 'Male', 'Turquoise', null, false, 'dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (493, 'Boler', 'mbolerdo@usnews.com', 'Male', 'Pink', null, true, 'enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (494, 'Brabyn', 'jbrabyndp@psu.edu', 'Male', 'Teal', null, false, 'posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (495, 'Eddowis', 'reddowisdq@wikispaces.com', 'Male', 'Maroon', null, true, 'nulla quisque arcu libero rutrum ac lobortis vel dapibus at diam nam tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (496, 'Coggan', 'gcoggandr@ning.com', 'Female', 'Mauv', null, false, 'suspendisse ornare consequat lectus in est risus auctor sed tristique in tempus sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (497, 'Gavigan', 'ggavigands@histats.com', 'Male', 'Violet', null, false, 'erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (498, 'Dyson', 'tdysondt@deviantart.com', 'Male', 'Purple', null, false, 'non ligula pellentesque ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (499, 'Cowpland', 'scowplanddu@archive.org', 'Male', 'Orange', null, true, 'vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (500, 'Megainey', 'pmegaineydv@simplemachines.org', 'Female', 'Purple', null, true, 'sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (501, 'Durham', 'mdurhamdw@slashdot.org', 'Female', 'Orange', null, true, 'faucibus cursus urna ut tellus nulla ut erat id mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (502, 'Orrice', 'forricedx@webeden.co.uk', 'Male', 'Fuscia', null, true, 'in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (503, 'Kinnerley', 'dkinnerleydy@unesco.org', 'Female', 'Turquoise', null, true, 'congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (504, 'McCurry', 'bmccurrydz@psu.edu', 'Female', 'Indigo', null, false, 'nulla ultrices aliquet maecenas leo odio condimentum id luctus nec molestie sed justo pellentesque viverra'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (505, 'Houldin', 'dhouldine0@about.me', 'Female', 'Indigo', null, true, 'ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (506, 'Guinane', 'hguinanee1@tmall.com', 'Female', 'Goldenrod', null, true, 'a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (507, 'Breckell', 'ibreckelle2@discovery.com', 'Female', 'Goldenrod', null, false, 'semper porta volutpat quam pede lobortis ligula sit amet eleifend pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (508, 'Raleston', 'wralestone3@devhub.com', 'Male', 'Violet', null, false, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam vitae quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (509, 'Wais', 'kwaise4@delicious.com', 'Male', 'Aquamarine', null, false, 'velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (510, 'Blannin', 'mblannine5@addthis.com', 'Male', 'Blue', null, false, 'at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (511, 'Chirm', 'wchirme6@desdev.cn', 'Male', 'Fuscia', null, true, 'amet turpis elementum ligula vehicula consequat morbi a ipsum integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (512, 'Janos', 'kjanose7@blog.com', 'Male', 'Teal', null, true, 'orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (513, 'O''Mullaney', 'momullaneye8@marketwatch.com', 'Female', 'Fuscia', null, false, 'nulla sed accumsan felis ut at dolor quis odio consequat varius integer ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (514, 'Beed', 'gbeede9@exblog.jp', 'Female', 'Crimson', null, false, 'ut nulla sed accumsan felis ut at dolor quis odio consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (515, 'Bony', 'lbonyea@cbc.ca', 'Male', 'Violet', null, true, 'quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (516, 'Danbi', 'jdanbieb@amazon.co.jp', 'Male', 'Puce', null, false, 'non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (517, 'Ubsdale', 'mubsdaleec@jalbum.net', 'Male', 'Blue', null, false, 'nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (518, 'Pettinger', 'npettingered@oracle.com', 'Female', 'Red', null, false, 'donec vitae nisi nam ultrices libero non mattis pulvinar nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (519, 'Greenhouse', 'mgreenhouseee@tiny.cc', 'Female', 'Yellow', null, true, 'aliquet ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (520, 'Spilstead', 'cspilsteadef@tinyurl.com', 'Male', 'Orange', null, false, 'lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (521, 'Moogan', 'smooganeg@i2i.jp', 'Female', 'Green', null, true, 'orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat tortor sollicitudin mi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (522, 'Sacchetti', 'gsacchettieh@tuttocitta.it', 'Male', 'Purple', null, false, 'mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper augue a suscipit nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (523, 'Godlonton', 'mgodlontonei@blog.com', 'Male', 'Khaki', null, false, 'et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (524, 'Filyukov', 'vfilyukovej@yellowbook.com', 'Male', 'Fuscia', null, true, 'sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (525, 'Rousby', 'srousbyek@sciencedaily.com', 'Female', 'Violet', null, false, 'dui maecenas tristique est et tempus semper est quam pharetra magna ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (526, 'Sail', 'rsailel@miitbeian.gov.cn', 'Female', 'Pink', null, true, 'quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (527, 'Mouland', 'fmoulandem@craigslist.org', 'Female', 'Red', null, true, 'id turpis integer aliquet massa id lobortis convallis tortor risus dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (528, 'Boij', 'sboijen@paginegialle.it', 'Female', 'Crimson', null, false, 'eros vestibulum ac est lacinia nisi venenatis tristique fusce congue diam id ornare imperdiet sapien urna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (529, 'Brockherst', 'ebrockhersteo@seattletimes.com', 'Female', 'Aquamarine', null, false, 'nisi vulputate nonummy maecenas tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (530, 'Baumert', 'fbaumertep@bigcartel.com', 'Male', 'Fuscia', null, true, 'lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu sapien cursus vestibulum proin eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (531, 'Pennicott', 'cpennicotteq@berkeley.edu', 'Female', 'Violet', null, false, 'in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem vitae mattis nibh'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (532, 'Kment', 'mkmenter@moonfruit.com', 'Female', 'Crimson', null, true, 'tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse ornare consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (533, 'Beazleigh', 'abeazleighes@seesaa.net', 'Male', 'Turquoise', null, false, 'est risus auctor sed tristique in tempus sit amet sem fusce consequat nulla nisl nunc nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (534, 'Lanceley', 'clanceleyet@usnews.com', 'Male', 'Crimson', null, true, 'vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis parturient'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (535, 'Pomfret', 'kpomfreteu@csmonitor.com', 'Male', 'Indigo', null, false, 'enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (536, 'Falcus', 'lfalcusev@imageshack.us', 'Female', 'Purple', null, true, 'integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (537, 'Bargery', 'kbargeryew@seattletimes.com', 'Male', 'Green', null, true, 'morbi a ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (538, 'Rawlingson', 'brawlingsonex@bbc.co.uk', 'Female', 'Indigo', null, false, 'habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (539, 'Callow', 'ocallowey@cyberchimps.com', 'Female', 'Fuscia', null, false, 'nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (540, 'Elwill', 'celwillez@jiathis.com', 'Male', 'Turquoise', null, false, 'quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (541, 'Onn', 'connf0@instagram.com', 'Female', 'Blue', null, true, 'cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (542, 'Middlemass', 'emiddlemassf1@archive.org', 'Female', 'Orange', null, true, 'praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (543, 'O''Logan', 'kologanf2@epa.gov', 'Female', 'Orange', null, true, 'augue a suscipit nulla elit ac nulla sed vel enim sit amet nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (544, 'Kilshaw', 'dkilshawf3@sakura.ne.jp', 'Male', 'Pink', null, true, 'sed nisl nunc rhoncus dui vel sem sed sagittis nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (545, 'Loveitt', 'nloveittf4@yahoo.co.jp', 'Male', 'Red', null, false, 'nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (546, 'Folbige', 'bfolbigef5@hexun.com', 'Male', 'Indigo', null, true, 'convallis eget eleifend luctus ultricies eu nibh quisque id justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (547, 'Barenskie', 'ebarenskief6@vistaprint.com', 'Male', 'Turquoise', null, false, 'blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (548, 'Tinklin', 'gtinklinf7@jalbum.net', 'Female', 'Maroon', null, true, 'diam vitae quam suspendisse potenti nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (549, 'Krolman', 'jkrolmanf8@nasa.gov', 'Female', 'Blue', null, true, 'ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (550, 'Lindman', 'clindmanf9@accuweather.com', 'Female', 'Crimson', null, false, 'in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (551, 'O''Suaird', 'kosuairdfa@woothemes.com', 'Female', 'Indigo', null, false, 'posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (552, 'Cafe', 'ccafefb@exblog.jp', 'Female', 'Puce', null, true, 'pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (553, 'Exelby', 'rexelbyfc@usda.gov', 'Male', 'Khaki', null, false, 'ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices aliquet maecenas leo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (554, 'MacColm', 'gmaccolmfd@baidu.com', 'Female', 'Yellow', null, false, 'faucibus accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (555, 'Haddleston', 'dhaddlestonfe@wsj.com', 'Male', 'Red', null, true, 'dapibus duis at velit eu est congue elementum in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (556, 'Dunsford', 'ldunsfordff@uol.com.br', 'Male', 'Mauv', null, true, 'integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse ornare consequat lectus in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (557, 'Raatz', 'braatzfg@tinyurl.com', 'Male', 'Blue', null, false, 'nunc commodo placerat praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (558, 'Rendell', 'srendellfh@cpanel.net', 'Female', 'Green', null, true, 'odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (559, 'McCahill', 'emccahillfi@wp.com', 'Female', 'Maroon', null, true, 'semper sapien a libero nam dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (560, 'Mechi', 'hmechifj@uol.com.br', 'Female', 'Mauv', null, true, 'nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus etiam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (561, 'Beincken', 'gbeinckenfk@hibu.com', 'Male', 'Fuscia', null, true, 'nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (562, 'Greenset', 'pgreensetfl@jiathis.com', 'Male', 'Puce', null, true, 'convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (563, 'Dorricott', 'mdorricottfm@berkeley.edu', 'Male', 'Pink', null, true, 'orci eget orci vehicula condimentum curabitur in libero ut massa volutpat convallis morbi odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (564, 'Chatelain', 'cchatelainfn@spotify.com', 'Female', 'Teal', null, true, 'purus eu magna vulputate luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (565, 'Abramchik', 'pabramchikfo@mapy.cz', 'Male', 'Teal', null, true, 'fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (566, 'Chastagnier', 'cchastagnierfp@yellowpages.com', 'Female', 'Red', null, true, 'cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (567, 'Greenshields', 'agreenshieldsfq@goo.ne.jp', 'Male', 'Orange', null, true, 'rutrum nulla nunc purus phasellus in felis donec semper sapien a libero nam dui proin leo odio porttitor id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (568, 'O''Dea', 'bodeafr@cocolog-nifty.com', 'Male', 'Mauv', null, false, 'cursus urna ut tellus nulla ut erat id mauris vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (569, 'Charker', 'scharkerfs@archive.org', 'Female', 'Indigo', null, true, 'justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (570, 'Iorizzo', 'ciorizzoft@bloglines.com', 'Female', 'Purple', null, true, 'erat volutpat in congue etiam justo etiam pretium iaculis justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (571, 'Rochford', 'erochfordfu@reuters.com', 'Female', 'Blue', null, true, 'ut nulla sed accumsan felis ut at dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (572, 'Maffione', 'cmaffionefv@amazon.de', 'Male', 'Turquoise', null, false, 'suscipit ligula in lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (573, 'Brenneke', 'abrennekefw@telegraph.co.uk', 'Female', 'Crimson', null, false, 'lobortis ligula sit amet eleifend pede libero quis orci nullam molestie nibh in lectus pellentesque at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (574, 'Dennick', 'odennickfx@skype.com', 'Female', 'Purple', null, false, 'in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (575, 'Balk', 'ebalkfy@quantcast.com', 'Male', 'Mauv', null, false, 'est donec odio justo sollicitudin ut suscipit a feugiat et eros vestibulum ac est lacinia nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (576, 'Dewhirst', 'ldewhirstfz@home.pl', 'Male', 'Teal', null, false, 'mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (577, 'Cannell', 'dcannellg0@unesco.org', 'Female', 'Turquoise', null, true, 'lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu sapien cursus vestibulum proin eu mi nulla ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (578, 'Petren', 'mpetreng1@engadget.com', 'Female', 'Turquoise', null, true, 'vestibulum aliquet ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (579, 'Di Giorgio', 'rdigiorgiog2@latimes.com', 'Male', 'Fuscia', null, false, 'erat eros viverra eget congue eget semper rutrum nulla nunc purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (580, 'Rookledge', 'jrookledgeg3@msn.com', 'Male', 'Khaki', null, false, 'habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut erat id mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (581, 'Webborn', 'bwebborng4@zimbio.com', 'Female', 'Turquoise', null, false, 'nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat morbi a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (582, 'Vearncomb', 'vvearncombg5@mapy.cz', 'Male', 'Aquamarine', null, true, 'porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (583, 'Bellay', 'ebellayg6@nasa.gov', 'Female', 'Teal', null, true, 'tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst aliquam augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (584, 'Lumsdale', 'clumsdaleg7@bandcamp.com', 'Female', 'Crimson', null, true, 'metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (585, 'Cagan', 'ecagang8@newyorker.com', 'Male', 'Fuscia', null, false, 'sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in congue etiam justo etiam pretium'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (586, 'Lamont', 'llamontg9@cam.ac.uk', 'Male', 'Crimson', null, true, 'nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (587, 'Killimister', 'kkillimisterga@wisc.edu', 'Male', 'Maroon', null, false, 'lorem ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (588, 'Wennington', 'fwenningtongb@amazon.com', 'Male', 'Puce', null, false, 'lectus vestibulum quam sapien varius ut blandit non interdum in ante vestibulum ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (589, 'Hysom', 'lhysomgc@weebly.com', 'Female', 'Green', null, false, 'maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices aliquet maecenas leo odio condimentum id luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (590, 'Fullstone', 'cfullstonegd@dot.gov', 'Male', 'Teal', null, true, 'donec ut mauris eget massa tempor convallis nulla neque libero convallis eget eleifend luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (591, 'Cristofolo', 'kcristofologe@abc.net.au', 'Male', 'Indigo', null, true, 'tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (592, 'Crossland', 'mcrosslandgf@sun.com', 'Male', 'Violet', null, true, 'nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (593, 'Castiglione', 'pcastiglionegg@blogs.com', 'Female', 'Teal', null, true, 'quam pharetra magna ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (594, 'Ianne', 'kiannegh@mlb.com', 'Male', 'Violet', null, true, 'eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien a libero nam dui proin leo odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (595, 'Munden', 'hmundengi@sina.com.cn', 'Male', 'Puce', null, false, 'eros viverra eget congue eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (596, 'Sinott', 'dsinottgj@cafepress.com', 'Male', 'Indigo', null, false, 'in consequat ut nulla sed accumsan felis ut at dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (597, 'Tressler', 'ttresslergk@bloomberg.com', 'Male', 'Orange', null, false, 'quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum praesent blandit lacinia erat vestibulum sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (598, 'Clohisey', 'bclohiseygl@printfriendly.com', 'Male', 'Crimson', null, false, 'leo odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at dolor quis odio consequat varius integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (599, 'Bohea', 'gboheagm@macromedia.com', 'Male', 'Pink', null, false, 'dolor sit amet consectetuer adipiscing elit proin risus praesent lectus vestibulum quam sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (600, 'Fearnsides', 'afearnsidesgn@freewebs.com', 'Male', 'Fuscia', null, true, 'vel lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (601, 'Simonsen', 'rsimonsengo@nyu.edu', 'Male', 'Purple', null, true, 'aliquet massa id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (602, 'Eastcott', 'aeastcottgp@reuters.com', 'Female', 'Turquoise', null, true, 'turpis nec euismod scelerisque quam turpis adipiscing lorem vitae mattis nibh ligula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (603, 'Scoterbosh', 'vscoterboshgq@jalbum.net', 'Male', 'Yellow', null, true, 'at velit eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (604, 'Gebbie', 'bgebbiegr@arstechnica.com', 'Male', 'Pink', null, true, 'non mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (605, 'Anthony', 'manthonygs@gov.uk', 'Female', 'Puce', null, false, 'ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (606, 'Salliss', 'tsallissgt@google.com.hk', 'Male', 'Turquoise', null, true, 'convallis eget eleifend luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (607, 'Tottem', 'ctottemgu@4shared.com', 'Male', 'Aquamarine', null, false, 'nulla ut erat id mauris vulputate elementum nullam varius nulla facilisi cras non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (608, 'Loton', 'mlotongv@privacy.gov.au', 'Male', 'Orange', null, false, 'at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum ac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (609, 'Garnson', 'ngarnsongw@comsenz.com', 'Male', 'Turquoise', null, false, 'congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (610, 'Wither', 'hwithergx@irs.gov', 'Female', 'Crimson', null, true, 'blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (611, 'Kenninghan', 'mkenninghangy@earthlink.net', 'Female', 'Indigo', null, true, 'vel accumsan tellus nisi eu orci mauris lacinia sapien quis libero nullam sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (612, 'Rupp', 'aruppgz@godaddy.com', 'Female', 'Khaki', null, true, 'mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (613, 'Wallworke', 'pwallworkeh0@csmonitor.com', 'Female', 'Green', null, true, 'ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (614, 'Balloch', 'fballochh1@nhs.uk', 'Male', 'Violet', null, false, 'sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (615, 'Philler', 'hphillerh2@unesco.org', 'Female', 'Green', null, true, 'posuere cubilia curae nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit a feugiat et eros vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (616, 'Penas', 'kpenash3@sina.com.cn', 'Female', 'Crimson', null, true, 'nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (617, 'Swenson', 'hswensonh4@weather.com', 'Male', 'Puce', null, true, 'mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam congue risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (618, 'Freddi', 'dfreddih5@ted.com', 'Female', 'Teal', null, false, 'vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (619, 'Rablan', 'rrablanh6@vkontakte.ru', 'Male', 'Crimson', null, false, 'erat vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (620, 'Bredgeland', 'gbredgelandh7@ustream.tv', 'Male', 'Aquamarine', null, true, 'elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (621, 'Hamfleet', 'jhamfleeth8@adobe.com', 'Male', 'Khaki', null, false, 'quam sapien varius ut blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (622, 'Corden', 'bcordenh9@chron.com', 'Male', 'Puce', null, true, 'turpis adipiscing lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at turpis a pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (623, 'Bullcock', 'pbullcockha@businessinsider.com', 'Female', 'Blue', null, true, 'dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (624, 'Leask', 'vleaskhb@opensource.org', 'Male', 'Indigo', null, true, 'magna at nunc commodo placerat praesent blandit nam nulla integer pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (625, 'Siley', 'ysileyhc@dedecms.com', 'Male', 'Indigo', null, true, 'sodales sed tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (626, 'Meeus', 'bmeeushd@ask.com', 'Female', 'Pink', null, true, 'et ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo sollicitudin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (627, 'Kohring', 'hkohringhe@themeforest.net', 'Female', 'Green', null, true, 'maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (628, 'Smitton', 'msmittonhf@youtube.com', 'Male', 'Khaki', null, true, 'congue eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien a libero nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (629, 'Grissett', 'bgrissetthg@wiley.com', 'Female', 'Goldenrod', null, false, 'vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (630, 'Basden', 'hbasdenhh@amazon.co.uk', 'Male', 'Turquoise', null, false, 'dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (631, 'Nutbeam', 'nnutbeamhi@ustream.tv', 'Male', 'Purple', null, false, 'justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (632, 'Smiley', 'gsmileyhj@chicagotribune.com', 'Male', 'Orange', null, false, 'nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis parturient montes nascetur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (633, 'Toulson', 'ptoulsonhk@mapy.cz', 'Male', 'Red', null, false, 'proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (634, 'Dine-Hart', 'gdineharthl@histats.com', 'Male', 'Goldenrod', null, true, 'bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (635, 'Kunert', 'dkunerthm@dailymotion.com', 'Female', 'Fuscia', null, false, 'in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere felis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (636, 'Brockie', 'abrockiehn@about.com', 'Male', 'Purple', null, true, 'nulla eget eros elementum pellentesque quisque porta volutpat erat quisque erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (637, 'Hovey', 'jhoveyho@cyberchimps.com', 'Female', 'Maroon', null, true, 'in porttitor pede justo eu massa donec dapibus duis at velit eu est congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (638, 'McClinton', 'cmcclintonhp@trellian.com', 'Male', 'Mauv', null, false, 'ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (639, 'Morbey', 'mmorbeyhq@feedburner.com', 'Male', 'Violet', null, false, 'in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (640, 'Margrie', 'pmargriehr@cyberchimps.com', 'Female', 'Fuscia', null, true, 'lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (641, 'Gealle', 'cgeallehs@wordpress.com', 'Male', 'Puce', null, true, 'luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (642, 'Eglaise', 'aeglaiseht@reuters.com', 'Female', 'Blue', null, true, 'in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (643, 'Sowrah', 'gsowrahhu@dailymail.co.uk', 'Male', 'Indigo', null, true, 'nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (644, 'Benterman', 'pbentermanhv@gov.uk', 'Female', 'Pink', null, true, 'praesent id massa id nisl venenatis lacinia aenean sit amet justo morbi ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (645, 'Gooddy', 'fgooddyhw@aboutads.info', 'Male', 'Yellow', null, false, 'pede justo eu massa donec dapibus duis at velit eu est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (646, 'Vannacci', 'kvannaccihx@upenn.edu', 'Male', 'Goldenrod', null, false, 'integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (647, 'Bosquet', 'sbosquethy@scientificamerican.com', 'Male', 'Orange', null, false, 'consequat varius integer ac leo pellentesque ultrices mattis odio donec vitae nisi nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (648, 'Dexter', 'ddexterhz@nature.com', 'Male', 'Khaki', null, true, 'molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (649, 'Fullick', 'jfullicki0@ebay.com', 'Female', 'Yellow', null, true, 'duis bibendum felis sed interdum venenatis turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (650, 'Calveley', 'bcalveleyi1@timesonline.co.uk', 'Male', 'Yellow', null, true, 'vel est donec odio justo sollicitudin ut suscipit a feugiat et eros vestibulum ac est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (651, 'Colledge', 'mcolledgei2@privacy.gov.au', 'Male', 'Turquoise', null, false, 'quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (652, 'Kornalik', 'akornaliki3@fastcompany.com', 'Female', 'Red', null, false, 'ut erat id mauris vulputate elementum nullam varius nulla facilisi cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (653, 'Wigfall', 'bwigfalli4@hao123.com', 'Female', 'Green', null, false, 'potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (654, 'Meaton', 'mmeatoni5@salon.com', 'Male', 'Green', null, false, 'dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (655, 'Huscroft', 'hhuscrofti6@intel.com', 'Female', 'Turquoise', null, false, 'varius ut blandit non interdum in ante vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (656, 'Gooddy', 'ngooddyi7@ustream.tv', 'Male', 'Goldenrod', null, true, 'vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (657, 'Davidovic', 'adavidovici8@ed.gov', 'Male', 'Yellow', null, true, 'bibendum morbi non quam nec dui luctus rutrum nulla tellus in sagittis dui vel nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (658, 'Dumingos', 'bdumingosi9@nih.gov', 'Male', 'Violet', null, false, 'justo sollicitudin ut suscipit a feugiat et eros vestibulum ac est lacinia nisi venenatis tristique fusce congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (659, 'Jemison', 'mjemisonia@businessweek.com', 'Female', 'Crimson', null, true, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (660, 'Morstatt', 'amorstattib@google.com.br', 'Male', 'Khaki', null, false, 'ultrices libero non mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel enim sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (661, 'Constanza', 'cconstanzaic@1und1.de', 'Male', 'Pink', null, true, 'diam erat fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (662, 'Oldershaw', 'toldershawid@ucla.edu', 'Female', 'Mauv', null, false, 'elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (663, 'Bollins', 'hbollinsie@house.gov', 'Female', 'Puce', null, true, 'pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (664, 'Sickling', 'dsicklingif@bizjournals.com', 'Male', 'Green', null, true, 'turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (665, 'Kincla', 'nkinclaig@people.com.cn', 'Male', 'Pink', null, false, 'sit amet turpis elementum ligula vehicula consequat morbi a ipsum integer a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (666, 'Wheadon', 'lwheadonih@bizjournals.com', 'Male', 'Blue', null, true, 'quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (667, 'Radolf', 'mradolfii@illinois.edu', 'Female', 'Crimson', null, true, 'diam id ornare imperdiet sapien urna pretium nisl ut volutpat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (668, 'Berthot', 'cberthotij@netvibes.com', 'Female', 'Orange', null, true, 'vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (669, 'Hanley', 'whanleyik@so-net.ne.jp', 'Female', 'Khaki', null, true, 'eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (670, 'Cerie', 'gcerieil@businessweek.com', 'Female', 'Maroon', null, true, 'ultrices posuere cubilia curae duis faucibus accumsan odio curabitur convallis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (671, 'Lusty', 'wlustyim@uol.com.br', 'Female', 'Yellow', null, true, 'quisque erat eros viverra eget congue eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (672, 'Bromfield', 'ebromfieldin@fc2.com', 'Male', 'Goldenrod', null, false, 'tortor quis turpis sed ante vivamus tortor duis mattis egestas metus aenean'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (673, 'Cardall', 'mcardallio@google.nl', 'Female', 'Aquamarine', null, false, 'vitae nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper augue a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (674, 'Paramor', 'eparamorip@dell.com', 'Female', 'Goldenrod', null, false, 'elit proin interdum mauris non ligula pellentesque ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (675, 'Cowlard', 'hcowlardiq@dmoz.org', 'Male', 'Pink', null, false, 'feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (676, 'Laugheran', 'rlaugheranir@hao123.com', 'Female', 'Fuscia', null, false, 'nullam varius nulla facilisi cras non velit nec nisi vulputate nonummy'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (677, 'MacIntosh', 'bmacintoshis@irs.gov', 'Male', 'Mauv', null, false, 'nam dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (678, 'Geary', 'lgearyit@apache.org', 'Male', 'Purple', null, false, 'proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (679, 'Sainer', 'msaineriu@cloudflare.com', 'Male', 'Orange', null, true, 'pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse ornare consequat lectus in est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (680, 'Spinnace', 'mspinnaceiv@histats.com', 'Male', 'Fuscia', null, false, 'eros vestibulum ac est lacinia nisi venenatis tristique fusce congue diam id ornare'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (681, 'Synnot', 'tsynnotiw@qq.com', 'Female', 'Mauv', null, false, 'sit amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (682, 'Whyatt', 'swhyattix@g.co', 'Female', 'Violet', null, false, 'donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (683, 'Pllu', 'bplluiy@prnewswire.com', 'Female', 'Violet', null, true, 'ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (684, 'Mannagh', 'umannaghiz@purevolume.com', 'Male', 'Goldenrod', null, true, 'massa volutpat convallis morbi odio odio elementum eu interdum eu tincidunt'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (685, 'Van den Velden', 'nvandenveldenj0@latimes.com', 'Female', 'Goldenrod', null, true, 'at velit eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (686, 'Amerighi', 'camerighij1@bloglovin.com', 'Male', 'Fuscia', null, false, 'eleifend luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (687, 'Riepel', 'ariepelj2@berkeley.edu', 'Male', 'Fuscia', null, true, 'arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (688, 'Falco', 'dfalcoj3@vistaprint.com', 'Female', 'Fuscia', null, false, 'eleifend pede libero quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (689, 'Pabel', 'spabelj4@epa.gov', 'Female', 'Yellow', null, false, 'vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (690, 'Olenikov', 'kolenikovj5@pagesperso-orange.fr', 'Male', 'Turquoise', null, true, 'adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (691, 'Sharratt', 'bsharrattj6@google.it', 'Male', 'Purple', null, false, 'vulputate luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (692, 'Massinger', 'omassingerj7@canalblog.com', 'Female', 'Yellow', null, false, 'velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (693, 'Joynson', 'ojoynsonj8@baidu.com', 'Female', 'Fuscia', null, false, 'vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget orci vehicula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (694, 'Landis', 'blandisj9@elpais.com', 'Female', 'Maroon', null, false, 'aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (695, 'Wyant', 'cwyantja@spiegel.de', 'Female', 'Purple', null, true, 'ligula nec sem duis aliquam convallis nunc proin at turpis a pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (696, 'Lideard', 'flideardjb@nsw.gov.au', 'Female', 'Puce', null, true, 'eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien a libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (697, 'Ovanesian', 'rovanesianjc@baidu.com', 'Female', 'Yellow', null, false, 'morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (698, 'O''Neil', 'doneiljd@wufoo.com', 'Female', 'Maroon', null, false, 'libero quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (699, 'Rivaland', 'trivalandje@wikipedia.org', 'Female', 'Fuscia', null, true, 'hac habitasse platea dictumst maecenas ut massa quis augue luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (700, 'Addlestone', 'aaddlestonejf@rambler.ru', 'Male', 'Puce', null, true, 'orci eget orci vehicula condimentum curabitur in libero ut massa volutpat convallis morbi odio odio elementum eu interdum eu tincidunt'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (701, 'Reveland', 'mrevelandjg@answers.com', 'Female', 'Crimson', null, false, 'in libero ut massa volutpat convallis morbi odio odio elementum eu interdum eu tincidunt in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (702, 'Blackaller', 'wblackallerjh@domainmarket.com', 'Female', 'Puce', null, false, 'morbi vel lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (703, 'Breckwell', 'ibreckwellji@sourceforge.net', 'Female', 'Red', null, false, 'in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce posuere felis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (704, 'Lappine', 'nlappinejj@soup.io', 'Male', 'Goldenrod', null, true, 'nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (705, 'Fabbro', 'mfabbrojk@usgs.gov', 'Male', 'Yellow', null, true, 'maecenas tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (706, 'Semper', 'vsemperjl@discovery.com', 'Male', 'Yellow', null, false, 'amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (707, 'Hinrichsen', 'bhinrichsenjm@domainmarket.com', 'Female', 'Green', null, true, 'aenean auctor gravida sem praesent id massa id nisl venenatis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (708, 'Beaconsall', 'jbeaconsalljn@wufoo.com', 'Female', 'Aquamarine', null, true, 'congue etiam justo etiam pretium iaculis justo in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (709, 'Compton', 'acomptonjo@w3.org', 'Male', 'Green', null, false, 'mi integer ac neque duis bibendum morbi non quam nec dui luctus rutrum nulla tellus in sagittis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (710, 'Gurton', 'rgurtonjp@vimeo.com', 'Male', 'Mauv', null, true, 'vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (711, 'Proudlove', 'mproudlovejq@mozilla.org', 'Male', 'Red', null, false, 'accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (712, 'Episcopio', 'hepiscopiojr@liveinternet.ru', 'Female', 'Turquoise', null, false, 'ullamcorper purus sit amet nulla quisque arcu libero rutrum ac lobortis vel dapibus at diam nam tristique tortor eu pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (713, 'Pedreschi', 'fpedreschijs@simplemachines.org', 'Female', 'Green', null, false, 'pretium quis lectus suspendisse potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (714, 'Tytherton', 'htythertonjt@gmpg.org', 'Female', 'Green', null, false, 'duis aliquam convallis nunc proin at turpis a pede posuere nonummy integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (715, 'Fairbanks', 'afairbanksju@miibeian.gov.cn', 'Female', 'Blue', null, false, 'bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt eu felis fusce'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (716, 'Donn', 'ldonnjv@bloglines.com', 'Male', 'Indigo', null, true, 'non lectus aliquam sit amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (717, 'Townsend', 'ltownsendjw@sbwire.com', 'Female', 'Turquoise', null, true, 'nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (718, 'Carrel', 'tcarreljx@symantec.com', 'Female', 'Purple', null, false, 'etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (719, 'Scotcher', 'escotcherjy@yellowpages.com', 'Male', 'Indigo', null, false, 'dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis parturient montes nascetur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (720, 'Winwood', 'gwinwoodjz@blog.com', 'Male', 'Violet', null, true, 'mattis pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (721, 'Kuzma', 'ekuzmak0@chicagotribune.com', 'Male', 'Crimson', null, true, 'ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer ac neque duis bibendum morbi non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (722, 'Triggs', 'mtriggsk1@go.com', 'Male', 'Crimson', null, true, 'feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (723, 'Gwinn', 'tgwinnk2@aol.com', 'Male', 'Goldenrod', null, true, 'dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (724, 'Sooper', 'lsooperk3@aol.com', 'Male', 'Indigo', null, false, 'quis lectus suspendisse potenti in eleifend quam a odio in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (725, 'Aizikov', 'daizikovk4@parallels.com', 'Male', 'Violet', null, false, 'interdum mauris non ligula pellentesque ultrices phasellus id sapien in sapien iaculis congue vivamus metus arcu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (726, 'Milland', 'smillandk5@123-reg.co.uk', 'Male', 'Khaki', null, true, 'phasellus sit amet erat nulla tempus vivamus in felis eu sapien cursus vestibulum proin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (727, 'Menel', 'mmenelk6@deviantart.com', 'Male', 'Puce', null, true, 'ut at dolor quis odio consequat varius integer ac leo pellentesque ultrices mattis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (728, 'Hallum', 'challumk7@cnn.com', 'Male', 'Yellow', null, false, 'aliquam erat volutpat in congue etiam justo etiam pretium iaculis justo in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (729, 'Wilman', 'swilmank8@ftc.gov', 'Male', 'Goldenrod', null, true, 'vestibulum sed magna at nunc commodo placerat praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget tempus vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (730, 'Swadon', 'pswadonk9@chicagotribune.com', 'Male', 'Maroon', null, true, 'dolor sit amet consectetuer adipiscing elit proin risus praesent lectus vestibulum quam sapien varius ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (731, 'Deppe', 'gdeppeka@umich.edu', 'Female', 'Red', null, false, 'proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (732, 'Giovannacci', 'mgiovannaccikb@loc.gov', 'Female', 'Crimson', null, true, 'nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (733, 'Jacmard', 'wjacmardkc@unicef.org', 'Female', 'Green', null, false, 'in congue etiam justo etiam pretium iaculis justo in hac'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (734, 'Attewill', 'rattewillkd@hhs.gov', 'Male', 'Mauv', null, false, 'rutrum nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at feugiat non pretium'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (735, 'McCarrison', 'cmccarrisonke@xinhuanet.com', 'Female', 'Indigo', null, true, 'ante vel ipsum praesent blandit lacinia erat vestibulum sed magna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (736, 'Batch', 'tbatchkf@bloglines.com', 'Male', 'Puce', null, true, 'consequat morbi a ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (737, 'Wynne', 'pwynnekg@mit.edu', 'Male', 'Maroon', null, false, 'lorem quisque ut erat curabitur gravida nisi at nibh in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (738, 'Flowerden', 'aflowerdenkh@constantcontact.com', 'Female', 'Yellow', null, false, 'quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (739, 'Gudyer', 'kgudyerki@mlb.com', 'Female', 'Fuscia', null, true, 'a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (740, 'Welham', 'bwelhamkj@comcast.net', 'Female', 'Pink', null, true, 'placerat praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget tempus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (741, 'Tuffin', 'ttuffinkk@free.fr', 'Female', 'Mauv', null, false, 'augue vel accumsan tellus nisi eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (742, 'Mosedall', 'lmosedallkl@google.co.uk', 'Male', 'Green', null, true, 'ut odio cras mi pede malesuada in imperdiet et commodo vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (743, 'Farloe', 'rfarloekm@liveinternet.ru', 'Female', 'Puce', null, false, 'lacus curabitur at ipsum ac tellus semper interdum mauris ullamcorper purus sit amet nulla quisque arcu libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (744, 'Pardue', 'iparduekn@instagram.com', 'Male', 'Aquamarine', null, true, 'suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (745, 'Gawler', 'sgawlerko@senate.gov', 'Male', 'Teal', null, false, 'quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (746, 'Mandry', 'hmandrykp@ning.com', 'Male', 'Maroon', null, false, 'blandit nam nulla integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (747, 'Laraway', 'rlarawaykq@cnn.com', 'Male', 'Red', null, false, 'porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (748, 'Dockreay', 'ddockreaykr@fda.gov', 'Male', 'Yellow', null, true, 'accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (749, 'Hallin', 'whallinks@forbes.com', 'Female', 'Crimson', null, false, 'tempor convallis nulla neque libero convallis eget eleifend luctus ultricies eu nibh quisque id justo sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (750, 'Rucklidge', 'drucklidgekt@ifeng.com', 'Female', 'Yellow', null, false, 'maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (751, 'Fresson', 'mfressonku@posterous.com', 'Female', 'Fuscia', null, false, 'at lorem integer tincidunt ante vel ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (752, 'Karlolak', 'nkarlolakkv@i2i.jp', 'Female', 'Green', null, false, 'justo sollicitudin ut suscipit a feugiat et eros vestibulum ac est lacinia nisi venenatis tristique fusce congue diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (753, 'Lagden', 'slagdenkw@blogspot.com', 'Male', 'Puce', null, false, 'pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (754, 'O''Kieran', 'pokierankx@tinyurl.com', 'Female', 'Fuscia', null, false, 'duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (755, 'Vennard', 'dvennardky@w3.org', 'Female', 'Turquoise', null, true, 'cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (756, 'Crankhorn', 'tcrankhornkz@microsoft.com', 'Female', 'Khaki', null, true, 'sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (757, 'Baxstare', 'mbaxstarel0@cam.ac.uk', 'Female', 'Pink', null, false, 'eget semper rutrum nulla nunc purus phasellus in felis donec semper sapien a libero nam dui proin leo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (758, 'Kilbourn', 'skilbournl1@redcross.org', 'Female', 'Purple', null, true, 'id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (759, 'Richt', 'trichtl2@slideshare.net', 'Male', 'Orange', null, false, 'sit amet erat nulla tempus vivamus in felis eu sapien cursus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (760, 'Eckery', 'aeckeryl3@constantcontact.com', 'Male', 'Violet', null, true, 'molestie sed justo pellentesque viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (761, 'Morcombe', 'nmorcombel4@networkadvertising.org', 'Female', 'Pink', null, true, 'sapien cursus vestibulum proin eu mi nulla ac enim in tempor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (762, 'Iori', 'lioril5@foxnews.com', 'Male', 'Blue', null, true, 'amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (763, 'Rantoull', 'jrantoulll6@ustream.tv', 'Male', 'Pink', null, false, 'ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (764, 'Guice', 'mguicel7@meetup.com', 'Male', 'Red', null, true, 'eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (765, 'Marwick', 'bmarwickl8@eventbrite.com', 'Female', 'Pink', null, false, 'ultrices posuere cubilia curae nulla dapibus dolor vel est donec odio justo sollicitudin ut suscipit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (766, 'Sendley', 'asendleyl9@godaddy.com', 'Male', 'Mauv', null, true, 'ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim lorem ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (767, 'Borzone', 'kborzonela@bravesites.com', 'Male', 'Mauv', null, true, 'sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (768, 'Durrad', 'gdurradlb@buzzfeed.com', 'Female', 'Indigo', null, false, 'pede morbi porttitor lorem id ligula suspendisse ornare consequat lectus in est risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (769, 'Oven', 'aovenlc@yale.edu', 'Female', 'Khaki', null, true, 'convallis morbi odio odio elementum eu interdum eu tincidunt in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (770, 'Surridge', 'gsurridgeld@dedecms.com', 'Female', 'Puce', null, false, 'magna ac consequat metus sapien ut nunc vestibulum ante ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (771, 'Caulket', 'mcaulketle@hao123.com', 'Female', 'Goldenrod', null, true, 'venenatis turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (772, 'Eastwell', 'eeastwelllf@chronoengine.com', 'Male', 'Goldenrod', null, true, 'sagittis sapien cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus etiam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (773, 'Readshall', 'nreadshalllg@homestead.com', 'Female', 'Indigo', null, true, 'at turpis donec posuere metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (774, 'Bridal', 'abridallh@trellian.com', 'Male', 'Crimson', null, true, 'vestibulum aliquet ultrices erat tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (775, 'Biss', 'tbissli@freewebs.com', 'Male', 'Orange', null, true, 'non interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (776, 'Folling', 'afollinglj@google.pl', 'Female', 'Yellow', null, true, 'vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (777, 'Stodd', 'astoddlk@meetup.com', 'Female', 'Puce', null, true, 'est risus auctor sed tristique in tempus sit amet sem fusce consequat nulla nisl nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (778, 'Eisak', 'jeisakll@oaic.gov.au', 'Male', 'Green', null, false, 'habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (779, 'Mealand', 'gmealandlm@deliciousdays.com', 'Female', 'Indigo', null, true, 'sit amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales sed tincidunt'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (780, 'Banbrook', 'dbanbrookln@bing.com', 'Female', 'Fuscia', null, true, 'aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (781, 'Giacomelli', 'jgiacomellilo@salon.com', 'Male', 'Mauv', null, false, 'ut erat id mauris vulputate elementum nullam varius nulla facilisi cras non velit nec nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (782, 'Balsellie', 'lbalsellielp@howstuffworks.com', 'Female', 'Aquamarine', null, false, 'sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (783, 'Kirwood', 'fkirwoodlq@dailymotion.com', 'Male', 'Fuscia', null, false, 'iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (784, 'Luxton', 'mluxtonlr@parallels.com', 'Female', 'Violet', null, false, 'amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (785, 'Cranke', 'ccrankels@wikispaces.com', 'Female', 'Turquoise', null, true, 'ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris non ligula pellentesque ultrices phasellus id sapien in sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (786, 'Jukubczak', 'kjukubczaklt@google.com.hk', 'Male', 'Aquamarine', null, true, 'arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (787, 'Savaage', 'gsavaagelu@webnode.com', 'Female', 'Red', null, true, 'imperdiet et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (788, 'Thowless', 'mthowlesslv@free.fr', 'Female', 'Violet', null, true, 'accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam fringilla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (789, 'Tolomio', 'mtolomiolw@washingtonpost.com', 'Male', 'Yellow', null, false, 'adipiscing lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at turpis a pede posuere'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (790, 'Woolvin', 'fwoolvinlx@de.vu', 'Female', 'Blue', null, true, 'nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (791, 'Janway', 'vjanwayly@hao123.com', 'Female', 'Puce', null, false, 'nibh in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (792, 'Phil', 'pphillz@studiopress.com', 'Female', 'Crimson', null, false, 'eu magna vulputate luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (793, 'Yeldon', 'dyeldonm0@themeforest.net', 'Male', 'Blue', null, false, 'felis sed interdum venenatis turpis enim blandit mi in porttitor pede justo eu massa'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (794, 'Noni', 'mnonim1@ocn.ne.jp', 'Female', 'Goldenrod', null, true, 'vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (795, 'Tolhurst', 'etolhurstm2@guardian.co.uk', 'Male', 'Goldenrod', null, true, 'tristique est et tempus semper est quam pharetra magna ac consequat metus sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (796, 'Maddams', 'lmaddamsm3@mlb.com', 'Male', 'Orange', null, true, 'posuere cubilia curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (797, 'Leborgne', 'jleborgnem4@answers.com', 'Male', 'Purple', null, true, 'nullam porttitor lacus at turpis donec posuere metus vitae ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (798, 'Kob', 'lkobm5@nih.gov', 'Female', 'Purple', null, true, 'et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit proin interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (799, 'McKmurrie', 'mmckmurriem6@bigcartel.com', 'Male', 'Turquoise', null, true, 'nec nisi volutpat eleifend donec ut dolor morbi vel lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (800, 'Vedekhov', 'avedekhovm7@bbc.co.uk', 'Male', 'Khaki', null, false, 'sit amet nulla quisque arcu libero rutrum ac lobortis vel dapibus at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (801, 'Lumsdon', 'mlumsdonm8@wunderground.com', 'Female', 'Goldenrod', null, false, 'integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (802, 'Slate', 'vslatem9@patch.com', 'Male', 'Purple', null, true, 'felis donec semper sapien a libero nam dui proin leo odio porttitor id consequat in consequat ut nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (803, 'McDell', 'emcdellma@boston.com', 'Female', 'Indigo', null, true, 'justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut erat id mauris vulputate elementum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (804, 'Bollard', 'sbollardmb@reverbnation.com', 'Male', 'Turquoise', null, true, 'venenatis turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at velit eu est congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (805, 'Tejero', 'stejeromc@typepad.com', 'Female', 'Turquoise', null, true, 'est phasellus sit amet erat nulla tempus vivamus in felis eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (806, 'Tilt', 'btiltmd@adobe.com', 'Male', 'Aquamarine', null, false, 'ac leo pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (807, 'Anthoine', 'canthoineme@shutterfly.com', 'Male', 'Pink', null, false, 'eu felis fusce posuere felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (808, 'Comber', 'ecombermf@tamu.edu', 'Male', 'Maroon', null, true, 'curae duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (809, 'Ribbens', 'gribbensmg@blogger.com', 'Female', 'Red', null, false, 'ante vel ipsum praesent blandit lacinia erat vestibulum sed magna'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (810, 'Callam', 'jcallammh@flavors.me', 'Male', 'Mauv', null, false, 'nam dui proin leo odio porttitor id consequat in consequat ut nulla sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (811, 'Kenvin', 'dkenvinmi@theatlantic.com', 'Male', 'Mauv', null, true, 'libero ut massa volutpat convallis morbi odio odio elementum eu interdum eu tincidunt in leo maecenas pulvinar'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (812, 'Flarity', 'bflaritymj@eepurl.com', 'Male', 'Yellow', null, false, 'nibh fusce lacus purus aliquet at feugiat non pretium quis lectus suspendisse potenti in eleifend'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (813, 'Summerill', 'jsummerillmk@wired.com', 'Male', 'Goldenrod', null, true, 'orci eget orci vehicula condimentum curabitur in libero ut massa volutpat convallis morbi odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (814, 'Burne', 'cburneml@webeden.co.uk', 'Male', 'Crimson', null, false, 'nulla integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula suspendisse ornare consequat lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (815, 'O''Howbane', 'aohowbanemm@accuweather.com', 'Female', 'Orange', null, false, 'lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at turpis a pede'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (816, 'De Goey', 'jdegoeymn@paginegialle.it', 'Male', 'Mauv', null, true, 'sapien placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (817, 'Luisetti', 'hluisettimo@tiny.cc', 'Female', 'Green', null, true, 'feugiat non pretium quis lectus suspendisse potenti in eleifend quam a odio in hac habitasse platea dictumst maecenas ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (818, 'Iacomelli', 'kiacomellimp@illinois.edu', 'Female', 'Orange', null, false, 'sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (819, 'Lourenco', 'slourencomq@behance.net', 'Male', 'Crimson', null, true, 'etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut erat id mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (820, 'Meas', 'cmeasmr@booking.com', 'Female', 'Khaki', null, true, 'et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat tortor sollicitudin mi sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (821, 'Andino', 'bandinoms@shinystat.com', 'Female', 'Red', null, false, 'in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (822, 'Bethune', 'cbethunemt@loc.gov', 'Male', 'Teal', null, false, 'cras pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (823, 'Castello', 'scastellomu@zdnet.com', 'Male', 'Aquamarine', null, false, 'lectus aliquam sit amet diam in magna bibendum imperdiet nullam orci pede venenatis non sodales'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (824, 'Romei', 'bromeimv@shareasale.com', 'Male', 'Indigo', null, true, 'pellentesque ultrices mattis odio donec vitae nisi nam ultrices libero'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (825, 'Scriver', 'fscrivermw@hubpages.com', 'Female', 'Mauv', null, false, 'scelerisque quam turpis adipiscing lorem vitae mattis nibh ligula nec sem duis aliquam convallis nunc proin at turpis a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (826, 'Whitten', 'jwhittenmx@latimes.com', 'Male', 'Goldenrod', null, false, 'metus sapien ut nunc vestibulum ante ipsum primis in faucibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (827, 'Moodycliffe', 'bmoodycliffemy@umich.edu', 'Male', 'Yellow', null, false, 'suspendisse potenti in eleifend quam a odio in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (828, 'MacLaig', 'kmaclaigmz@house.gov', 'Male', 'Mauv', null, true, 'in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (829, 'Nussgen', 'onussgenn0@examiner.com', 'Female', 'Green', null, true, 'lectus in quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (830, 'Blewmen', 'ublewmenn1@is.gd', 'Female', 'Maroon', null, false, 'id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (831, 'Tomasek', 'ttomasekn2@noaa.gov', 'Male', 'Puce', null, false, 'sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (832, 'McCarroll', 'jmccarrolln3@paypal.com', 'Female', 'Purple', null, false, 'faucibus orci luctus et ultrices posuere cubilia curae duis faucibus accumsan odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (833, 'Avrahamian', 'lavrahamiann4@twitter.com', 'Female', 'Puce', null, true, 'vel est donec odio justo sollicitudin ut suscipit a feugiat et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (834, 'Shovel', 'sshoveln5@ox.ac.uk', 'Male', 'Aquamarine', null, false, 'auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (835, 'Sewell', 'nsewelln6@homestead.com', 'Female', 'Violet', null, true, 'id massa id nisl venenatis lacinia aenean sit amet justo morbi ut odio cras mi pede malesuada in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (836, 'Tarquinio', 'mtarquinion7@seesaa.net', 'Male', 'Turquoise', null, true, 'a ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus morbi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (837, 'Wallbanks', 'mwallbanksn8@pbs.org', 'Female', 'Yellow', null, false, 'felis fusce posuere felis sed lacus morbi sem mauris laoreet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (838, 'Frankom', 'mfrankomn9@mail.ru', 'Male', 'Violet', null, true, 'pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla sed vel enim sit amet nunc viverra dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (839, 'Stitwell', 'astitwellna@miitbeian.gov.cn', 'Female', 'Red', null, false, 'pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (840, 'Morling', 'tmorlingnb@bigcartel.com', 'Female', 'Khaki', null, true, 'elementum ligula vehicula consequat morbi a ipsum integer a nibh in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (841, 'Oaker', 'toakernc@irs.gov', 'Male', 'Crimson', null, true, 'a pede posuere nonummy integer non velit donec diam neque vestibulum eget'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (842, 'Nerger', 'gnergernd@springer.com', 'Male', 'Violet', null, true, 'odio donec vitae nisi nam ultrices libero non mattis pulvinar nulla pede ullamcorper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (843, 'Verbruggen', 'mverbruggenne@sfgate.com', 'Female', 'Maroon', null, true, 'nascetur ridiculus mus etiam vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (844, 'Layman', 'blaymannf@edublogs.org', 'Male', 'Purple', null, false, 'quisque arcu libero rutrum ac lobortis vel dapibus at diam nam tristique tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (845, 'Greenslade', 'mgreensladeng@cdc.gov', 'Male', 'Blue', null, false, 'in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel ipsum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (846, 'Devenport', 'ldevenportnh@4shared.com', 'Male', 'Goldenrod', null, true, 'ut blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (847, 'Lay', 'clayni@aol.com', 'Female', 'Mauv', null, true, 'pellentesque eget nunc donec quis orci eget orci vehicula condimentum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (848, 'Enriques', 'lenriquesnj@cbslocal.com', 'Female', 'Violet', null, true, 'tortor risus dapibus augue vel accumsan tellus nisi eu orci mauris lacinia sapien quis libero nullam sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (849, 'Spry', 'nsprynk@wired.com', 'Male', 'Aquamarine', null, false, 'id lobortis convallis tortor risus dapibus augue vel accumsan tellus nisi'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (850, 'Vasey', 'fvaseynl@vk.com', 'Male', 'Purple', null, true, 'nunc vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae mauris viverra diam vitae'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (851, 'Stiegar', 'kstiegarnm@google.pl', 'Male', 'Orange', null, true, 'in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (852, 'Palmer', 'mpalmernn@plala.or.jp', 'Male', 'Crimson', null, false, 'amet eleifend pede libero quis orci nullam molestie nibh in lectus pellentesque at nulla suspendisse potenti cras in purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (853, 'Rivallant', 'jrivallantno@timesonline.co.uk', 'Female', 'Indigo', null, true, 'et magnis dis parturient montes nascetur ridiculus mus etiam vel augue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (854, 'Nugent', 'enugentnp@addthis.com', 'Male', 'Pink', null, true, 'leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (855, 'Tuttle', 'gtuttlenq@slashdot.org', 'Female', 'Orange', null, true, 'sit amet justo morbi ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (856, 'Zorzi', 'mzorzinr@mlb.com', 'Male', 'Teal', null, false, 'elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (857, 'Speenden', 'rspeendenns@ocn.ne.jp', 'Female', 'Mauv', null, false, 'justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem id ligula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (858, 'Haws', 'mhawsnt@cargocollective.com', 'Female', 'Yellow', null, false, 'ac est lacinia nisi venenatis tristique fusce congue diam id ornare imperdiet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (859, 'Allott', 'hallottnu@discovery.com', 'Female', 'Khaki', null, true, 'eu sapien cursus vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (860, 'Bilston', 'lbilstonnv@vimeo.com', 'Male', 'Yellow', null, false, 'pretium quis lectus suspendisse potenti in eleifend quam a odio in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (861, 'Fairney', 'nfairneynw@si.edu', 'Male', 'Turquoise', null, false, 'sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (862, 'Fauguel', 'ifauguelnx@nifty.com', 'Female', 'Yellow', null, true, 'curae mauris viverra diam vitae quam suspendisse potenti nullam porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (863, 'Awde', 'pawdeny@weebly.com', 'Male', 'Fuscia', null, false, 'id massa id nisl venenatis lacinia aenean sit amet justo morbi ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (864, 'Scading', 'mscadingnz@ehow.com', 'Female', 'Orange', null, true, 'velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (865, 'Ainslie', 'hainslieo0@networksolutions.com', 'Female', 'Teal', null, false, 'ligula vehicula consequat morbi a ipsum integer a nibh in quis justo maecenas rhoncus aliquam lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (866, 'Gatley', 'agatleyo1@chicagotribune.com', 'Male', 'Fuscia', null, true, 'vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (867, 'Lednor', 'olednoro2@nature.com', 'Male', 'Violet', null, false, 'vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (868, 'Braithwait', 'jbraithwaito3@netlog.com', 'Female', 'Purple', null, false, 'nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (869, 'Moubray', 'cmoubrayo4@epa.gov', 'Male', 'Yellow', null, true, 'tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer ac neque duis bibendum morbi non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (870, 'Casoni', 'rcasonio5@techcrunch.com', 'Male', 'Red', null, true, 'proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque quam turpis adipiscing lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (871, 'Mathe', 'mmatheo6@desdev.cn', 'Male', 'Orange', null, true, 'ac neque duis bibendum morbi non quam nec dui luctus rutrum nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (872, 'Kiebes', 'dkiebeso7@gov.uk', 'Male', 'Fuscia', null, false, 'gravida nisi at nibh in hac habitasse platea dictumst aliquam augue quam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (873, 'Godrich', 'ngodricho8@unc.edu', 'Male', 'Aquamarine', null, false, 'eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (874, 'Coast', 'mcoasto9@scientificamerican.com', 'Male', 'Mauv', null, false, 'blandit lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (875, 'Grimsell', 'sgrimselloa@forbes.com', 'Male', 'Indigo', null, false, 'pulvinar nulla pede ullamcorper augue a suscipit nulla elit ac nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (876, 'Scrange', 'wscrangeob@mit.edu', 'Female', 'Teal', null, false, 'morbi quis tortor id nulla ultrices aliquet maecenas leo odio condimentum id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (877, 'Maleby', 'dmalebyoc@com.com', 'Female', 'Teal', null, true, 'interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (878, 'Mathew', 'amathewod@w3.org', 'Female', 'Maroon', null, true, 'luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (879, 'Crummey', 'acrummeyoe@pen.io', 'Male', 'Turquoise', null, true, 'et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet consectetuer adipiscing elit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (880, 'O''Day', 'codayof@sphinn.com', 'Male', 'Red', null, false, 'maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (881, 'Goodson', 'agoodsonog@twitpic.com', 'Female', 'Indigo', null, false, 'nulla nunc purus phasellus in felis donec semper sapien a libero nam dui proin leo odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (882, 'Shapiro', 'hshapirooh@amazon.de', 'Female', 'Purple', null, true, 'elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (883, 'Alesi', 'balesioi@yellowpages.com', 'Female', 'Maroon', null, true, 'dolor vel est donec odio justo sollicitudin ut suscipit a feugiat et eros vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (884, 'Whitrod', 'cwhitrodoj@amazon.de', 'Male', 'Crimson', null, false, 'in est risus auctor sed tristique in tempus sit amet sem fusce consequat nulla nisl nunc nisl duis bibendum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (885, 'Memmory', 'omemmoryok@wikia.com', 'Female', 'Blue', null, true, 'posuere nonummy integer non velit donec diam neque vestibulum eget vulputate ut ultrices vel augue vestibulum ante'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (886, 'Foyston', 'bfoystonol@bbb.org', 'Male', 'Mauv', null, true, 'felis donec semper sapien a libero nam dui proin leo odio'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (887, 'Riddler', 'nriddlerom@gmpg.org', 'Male', 'Green', null, true, 'ipsum dolor sit amet consectetuer adipiscing elit proin interdum mauris non ligula'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (888, 'Sine', 'xsineon@google.es', 'Male', 'Indigo', null, true, 'turpis eget elit sodales scelerisque mauris sit amet eros suspendisse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (889, 'Leopard', 'eleopardoo@seesaa.net', 'Female', 'Mauv', null, true, 'turpis elementum ligula vehicula consequat morbi a ipsum integer a nibh in quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (890, 'Zuann', 'azuannop@redcross.org', 'Female', 'Crimson', null, true, 'bibendum felis sed interdum venenatis turpis enim blandit mi in porttitor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (891, 'Bartoszewski', 'gbartoszewskioq@washington.edu', 'Female', 'Puce', null, false, 'lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat morbi a ipsum integer a nibh in quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (892, 'Trainer', 'ftraineror@sfgate.com', 'Female', 'Fuscia', null, true, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (893, 'McAsgill', 'lmcasgillos@topsy.com', 'Female', 'Crimson', null, false, 'sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas metus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (894, 'Durant', 'ndurantot@reuters.com', 'Female', 'Violet', null, true, 'eu orci mauris lacinia sapien quis libero nullam sit amet turpis elementum ligula vehicula consequat morbi a ipsum integer a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (895, 'Vasey', 'kvaseyou@mozilla.com', 'Male', 'Turquoise', null, false, 'tincidunt lacus at velit vivamus vel nulla eget eros elementum pellentesque quisque porta volutpat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (896, 'Eleshenar', 'releshenarov@blog.com', 'Male', 'Teal', null, false, 'rutrum nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus purus aliquet at feugiat non pretium quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (897, 'Wesker', 'eweskerow@instagram.com', 'Male', 'Green', null, false, 'luctus rutrum nulla tellus in sagittis dui vel nisl duis ac nibh fusce lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (898, 'Culcheth', 'aculchethox@list-manage.com', 'Male', 'Fuscia', null, false, 'aliquam non mauris morbi non lectus aliquam sit amet diam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (899, 'Stolte', 'fstolteoy@prweb.com', 'Female', 'Orange', null, true, 'at nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (900, 'Santus', 'msantusoz@sbwire.com', 'Male', 'Indigo', null, false, 'non velit nec nisi vulputate nonummy maecenas tincidunt lacus at velit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (901, 'Habbijam', 'mhabbijamp0@google.com.au', 'Female', 'Red', null, false, 'parturient montes nascetur ridiculus mus etiam vel augue vestibulum rutrum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (902, 'New', 'vnewp1@si.edu', 'Male', 'Turquoise', null, false, 'diam id ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (903, 'McIllroy', 'dmcillroyp2@guardian.co.uk', 'Female', 'Mauv', null, false, 'libero nullam sit amet turpis elementum ligula vehicula consequat morbi a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (904, 'Woltering', 'owolteringp3@fastcompany.com', 'Female', 'Crimson', null, true, 'tristique fusce congue diam id ornare imperdiet sapien urna pretium nisl ut volutpat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (905, 'Allans', 'callansp4@skyrock.com', 'Male', 'Teal', null, false, 'felis sed lacus morbi sem mauris laoreet ut rhoncus aliquet pulvinar sed nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (906, 'Jachimak', 'ljachimakp5@youku.com', 'Female', 'Turquoise', null, false, 'turpis enim blandit mi in porttitor pede justo eu massa donec dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (907, 'Foskett', 'hfoskettp6@scientificamerican.com', 'Male', 'Pink', null, false, 'mauris enim leo rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (908, 'Lewcock', 'dlewcockp7@artisteer.com', 'Female', 'Fuscia', null, false, 'dolor sit amet consectetuer adipiscing elit proin risus praesent lectus vestibulum quam sapien varius ut blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (909, 'Brumbie', 'abrumbiep8@google.ca', 'Male', 'Maroon', null, false, 'non pretium quis lectus suspendisse potenti in eleifend quam a'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (910, 'Saunper', 'msaunperp9@squarespace.com', 'Female', 'Fuscia', null, true, 'etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus nulla ut erat id mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (911, 'Readmire', 'hreadmirepa@state.gov', 'Male', 'Teal', null, true, 'pellentesque quisque porta volutpat erat quisque erat eros viverra eget congue eget semper rutrum nulla nunc purus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (912, 'Simonyi', 'csimonyipb@icq.com', 'Female', 'Puce', null, false, 'in eleifend quam a odio in hac habitasse platea dictumst maecenas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (913, 'Blick', 'hblickpc@bluehost.com', 'Male', 'Red', null, true, 'placerat ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (914, 'Spencelayh', 'lspencelayhpd@soundcloud.com', 'Female', 'Violet', null, false, 'in felis donec semper sapien a libero nam dui proin leo odio porttitor id consequat in consequat ut nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (915, 'Senior', 'aseniorpe@example.com', 'Male', 'Blue', null, false, 'etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam faucibus cursus urna ut tellus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (916, 'Takos', 'ctakospf@jimdo.com', 'Female', 'Indigo', null, false, 'quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (917, 'Ashbee', 'vashbeepg@usa.gov', 'Female', 'Violet', null, false, 'maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices aliquet maecenas leo odio condimentum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (918, 'Pabelik', 'ppabelikph@elpais.com', 'Female', 'Turquoise', null, false, 'quam pede lobortis ligula sit amet eleifend pede libero quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (919, 'Antoinet', 'dantoinetpi@hc360.com', 'Male', 'Khaki', null, false, 'vivamus metus arcu adipiscing molestie hendrerit at vulputate vitae nisl aenean lectus pellentesque eget nunc donec quis orci eget orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (920, 'Wieprecht', 'owieprechtpj@php.net', 'Female', 'Violet', null, true, 'auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit amet justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (921, 'Gallego', 'ygallegopk@dmoz.org', 'Female', 'Fuscia', null, true, 'eget eros elementum pellentesque quisque porta volutpat erat quisque erat eros viverra eget congue'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (922, 'Greenhill', 'tgreenhillpl@craigslist.org', 'Male', 'Crimson', null, true, 'lacinia erat vestibulum sed magna at nunc commodo placerat praesent blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (923, 'Top', 'jtoppm@webnode.com', 'Male', 'Fuscia', null, false, 'duis bibendum morbi non quam nec dui luctus rutrum nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (924, 'Jeanes', 'hjeanespn@over-blog.com', 'Male', 'Pink', null, true, 'eget elit sodales scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus tortor duis mattis egestas'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (925, 'Marks', 'dmarkspo@freewebs.com', 'Female', 'Yellow', null, false, 'vitae ipsum aliquam non mauris morbi non lectus aliquam sit amet diam in magna bibendum imperdiet nullam orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (926, 'Carp', 'gcarppp@123-reg.co.uk', 'Male', 'Crimson', null, false, 'enim sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (927, 'Haward', 'lhawardpq@istockphoto.com', 'Male', 'Indigo', null, true, 'vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis dis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (928, 'Scrannage', 'mscrannagepr@arizona.edu', 'Female', 'Goldenrod', null, false, 'pede malesuada in imperdiet et commodo vulputate justo in blandit ultrices enim lorem ipsum dolor sit amet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (929, 'Pfertner', 'rpfertnerps@soup.io', 'Female', 'Goldenrod', null, true, 'ut erat id mauris vulputate elementum nullam varius nulla facilisi cras non velit nec nisi vulputate nonummy maecenas tincidunt lacus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (930, 'Fulger', 'wfulgerpt@yahoo.co.jp', 'Male', 'Khaki', null, true, 'arcu sed augue aliquam erat volutpat in congue etiam justo etiam pretium iaculis justo in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (931, 'Grolmann', 'hgrolmannpu@twitter.com', 'Female', 'Indigo', null, false, 'nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt ante vel'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (932, 'Le Sieur', 'clesieurpv@jigsy.com', 'Female', 'Green', null, false, 'in congue etiam justo etiam pretium iaculis justo in hac habitasse platea dictumst etiam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (933, 'Coull', 'tcoullpw@slashdot.org', 'Male', 'Aquamarine', null, true, 'consequat in consequat ut nulla sed accumsan felis ut at dolor quis odio consequat varius integer ac leo pellentesque ultrices'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (934, 'Sprasen', 'tsprasenpx@nymag.com', 'Male', 'Blue', null, false, 'accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (935, 'Redit', 'kreditpy@state.gov', 'Female', 'Red', null, true, 'fusce consequat nulla nisl nunc nisl duis bibendum felis sed interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (936, 'Dudden', 'dduddenpz@wired.com', 'Female', 'Khaki', null, true, 'ultricies eu nibh quisque id justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (937, 'Tritten', 'htrittenq0@over-blog.com', 'Female', 'Purple', null, true, 'luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (938, 'Kinnier', 'akinnierq1@prweb.com', 'Male', 'Indigo', null, true, 'urna ut tellus nulla ut erat id mauris vulputate elementum nullam varius nulla'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (939, 'Vannozzii', 'ivannozziiq2@ucoz.ru', 'Male', 'Yellow', null, false, 'rhoncus sed vestibulum sit amet cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus augue vel accumsan'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (940, 'Chumley', 'gchumleyq3@rakuten.co.jp', 'Female', 'Red', null, false, 'luctus cum sociis natoque penatibus et magnis dis parturient montes nascetur ridiculus mus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (941, 'Cutting', 'jcuttingq4@pcworld.com', 'Male', 'Indigo', null, false, 'tincidunt nulla mollis molestie lorem quisque ut erat curabitur gravida nisi at nibh in hac habitasse platea dictumst'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (942, 'McClintock', 'smcclintockq5@usda.gov', 'Male', 'Blue', null, false, 'interdum eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (943, 'Wyper', 'jwyperq6@boston.com', 'Male', 'Khaki', null, false, 'quis justo maecenas rhoncus aliquam lacus morbi quis tortor id nulla ultrices aliquet maecenas leo odio condimentum id luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (944, 'Cullon', 'mcullonq7@nyu.edu', 'Male', 'Khaki', null, true, 'orci pede venenatis non sodales sed tincidunt eu felis fusce posuere felis sed lacus morbi sem mauris laoreet ut'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (945, 'Withur', 'jwithurq8@simplemachines.org', 'Female', 'Mauv', null, false, 'ac nulla sed vel enim sit amet nunc viverra dapibus nulla suscipit ligula in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (946, 'Sherred', 'asherredq9@quantcast.com', 'Male', 'Yellow', null, true, 'tempus sit amet sem fusce consequat nulla nisl nunc nisl duis bibendum felis sed'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (947, 'Kastel', 'mkastelqa@t.co', 'Female', 'Violet', null, false, 'faucibus orci luctus et ultrices posuere cubilia curae donec pharetra magna vestibulum aliquet ultrices erat'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (948, 'Stilwell', 'astilwellqb@intel.com', 'Female', 'Mauv', null, false, 'justo sit amet sapien dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (949, 'Rimell', 'crimellqc@aol.com', 'Male', 'Aquamarine', null, false, 'ante nulla justo aliquam quis turpis eget elit sodales scelerisque mauris'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (950, 'Goodchild', 'egoodchildqd@reuters.com', 'Female', 'Yellow', null, true, 'sit amet nunc viverra dapibus nulla suscipit ligula in lacus curabitur at ipsum ac tellus semper interdum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (951, 'Fredi', 'kfrediqe@comcast.net', 'Male', 'Red', null, true, 'porttitor lacus at turpis donec posuere metus vitae ipsum aliquam non mauris morbi non lectus aliquam sit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (952, 'Adamson', 'ladamsonqf@reference.com', 'Male', 'Maroon', null, true, 'at velit eu est congue elementum in hac habitasse platea'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (953, 'MacCorkell', 'tmaccorkellqg@naver.com', 'Male', 'Green', null, false, 'quam pharetra magna ac consequat metus sapien ut nunc vestibulum ante ipsum primis in faucibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (954, 'Giorgioni', 'fgiorgioniqh@goo.gl', 'Male', 'Aquamarine', null, false, 'nibh in hac habitasse platea dictumst aliquam augue quam sollicitudin vitae consectetuer eget rutrum at lorem integer tincidunt'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (955, 'Bridgwood', 'fbridgwoodqi@cbsnews.com', 'Female', 'Pink', null, false, 'viverra pede ac diam cras pellentesque volutpat dui maecenas tristique est et tempus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (956, 'O'' Cloney', 'gocloneyqj@buzzfeed.com', 'Male', 'Red', null, false, 'interdum mauris ullamcorper purus sit amet nulla quisque arcu libero rutrum ac lobortis vel dapibus at diam nam tristique tortor'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (957, 'Jery', 'djeryqk@so-net.ne.jp', 'Female', 'Pink', null, false, 'fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (958, 'Di Napoli', 'bdinapoliql@is.gd', 'Female', 'Goldenrod', null, true, 'sed augue aliquam erat volutpat in congue etiam justo etiam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (959, 'Petris', 'rpetrisqm@diigo.com', 'Male', 'Maroon', null, true, 'eu est congue elementum in hac habitasse platea dictumst morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (960, 'Blacklidge', 'cblacklidgeqn@google.co.uk', 'Female', 'Fuscia', null, true, 'praesent blandit nam nulla integer pede justo lacinia eget tincidunt eget tempus vel pede morbi porttitor lorem'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (961, 'Rawlence', 'arawlenceqo@who.int', 'Female', 'Puce', null, false, 'semper est quam pharetra magna ac consequat metus sapien ut nunc vestibulum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (962, 'Bunclark', 'wbunclarkqp@google.co.jp', 'Male', 'Yellow', null, false, 'praesent id massa id nisl venenatis lacinia aenean sit amet justo morbi ut odio cras mi pede malesuada in imperdiet'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (963, 'Uzielli', 'ruzielliqq@scribd.com', 'Female', 'Red', null, true, 'tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla tempus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (964, 'Gartside', 'hgartsideqr@dailymotion.com', 'Male', 'Green', null, false, 'quam fringilla rhoncus mauris enim leo rhoncus sed vestibulum sit amet cursus id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (965, 'Matley', 'cmatleyqs@sciencedaily.com', 'Male', 'Indigo', null, true, 'tortor sollicitudin mi sit amet lobortis sapien sapien non mi integer ac neque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (966, 'Baggett', 'gbaggettqt@fotki.com', 'Male', 'Goldenrod', null, true, 'cursus id turpis integer aliquet massa id lobortis convallis tortor risus dapibus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (967, 'MacKill', 'mmackillqu@newsvine.com', 'Male', 'Purple', null, true, 'nulla facilisi cras non velit nec nisi vulputate nonummy maecenas tincidunt lacus at velit vivamus vel nulla eget eros'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (968, 'Punch', 'cpunchqv@ucoz.ru', 'Male', 'Crimson', null, false, 'ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (969, 'Haney', 'chaneyqw@seattletimes.com', 'Male', 'Green', null, false, 'dignissim vestibulum vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae nulla dapibus dolor vel est'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (970, 'Nelle', 'nnelleqx@dyndns.org', 'Female', 'Red', null, true, 'duis aliquam convallis nunc proin at turpis a pede posuere nonummy integer non velit donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (971, 'Brookson', 'sbrooksonqy@yellowbook.com', 'Female', 'Turquoise', null, false, 'morbi vestibulum velit id pretium iaculis diam erat fermentum justo nec condimentum neque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (972, 'Maso', 'lmasoqz@cnet.com', 'Female', 'Red', null, false, 'sit amet consectetuer adipiscing elit proin risus praesent lectus vestibulum quam sapien varius ut blandit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (973, 'Garmons', 'xgarmonsr0@webs.com', 'Male', 'Violet', null, false, 'duis faucibus accumsan odio curabitur convallis duis consequat dui nec nisi volutpat eleifend donec ut dolor morbi vel lectus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (974, 'Lanchester', 'clanchesterr1@dropbox.com', 'Female', 'Purple', null, false, 'in quis justo maecenas rhoncus aliquam lacus morbi quis tortor id'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (975, 'Cordoba', 'ecordobar2@senate.gov', 'Male', 'Red', null, true, 'maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (976, 'Luxmoore', 'dluxmoorer3@mlb.com', 'Female', 'Teal', null, true, 'dui proin leo odio porttitor id consequat in consequat ut nulla sed accumsan felis ut at dolor quis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (977, 'Terlinden', 'mterlindenr4@xinhuanet.com', 'Female', 'Blue', null, true, 'sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec ut mauris eget massa tempor convallis nulla neque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (978, 'Jozwiak', 'mjozwiakr5@clickbank.net', 'Female', 'Violet', null, false, 'mauris laoreet ut rhoncus aliquet pulvinar sed nisl nunc rhoncus dui vel sem sed sagittis nam congue risus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (979, 'Goatcher', 'bgoatcherr6@cbslocal.com', 'Female', 'Yellow', null, true, 'ante vel ipsum praesent blandit lacinia erat vestibulum sed magna at nunc commodo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (980, 'Bencher', 'tbencherr7@blogtalkradio.com', 'Male', 'Indigo', null, true, 'lectus pellentesque eget nunc donec quis orci eget orci vehicula condimentum curabitur in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (981, 'Zanolli', 'wzanollir8@ask.com', 'Female', 'Orange', null, true, 'venenatis lacinia aenean sit amet justo morbi ut odio cras mi pede malesuada in imperdiet et commodo vulputate justo'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (982, 'Lowings', 'hlowingsr9@kickstarter.com', 'Female', 'Violet', null, false, 'neque duis bibendum morbi non quam nec dui luctus rutrum nulla tellus in sagittis dui vel nisl'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (983, 'Kelberman', 'dkelbermanra@g.co', 'Male', 'Teal', null, true, 'luctus cum sociis natoque penatibus et magnis dis parturient montes'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (984, 'Forrington', 'mforringtonrb@newyorker.com', 'Male', 'Khaki', null, true, 'sed ante vivamus tortor duis mattis egestas metus aenean fermentum donec'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (985, 'Mourgue', 'amourguerc@state.tx.us', 'Male', 'Indigo', null, false, 'sem duis aliquam convallis nunc proin at turpis a pede posuere nonummy integer non'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (986, 'Hobbema', 'jhobbemard@foxnews.com', 'Female', 'Orange', null, false, 'volutpat in congue etiam justo etiam pretium iaculis justo in hac habitasse'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (987, 'Strugnell', 'dstrugnellre@nbcnews.com', 'Female', 'Goldenrod', null, true, 'ornare imperdiet sapien urna pretium nisl ut volutpat sapien arcu sed augue aliquam erat volutpat in'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (988, 'Rudham', 'hrudhamrf@wufoo.com', 'Male', 'Khaki', null, true, 'natoque penatibus et magnis dis parturient montes nascetur ridiculus mus vivamus vestibulum sagittis sapien cum sociis natoque penatibus et magnis'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (989, 'Rushe', 'grusherg@cafepress.com', 'Male', 'Goldenrod', null, true, 'id pretium iaculis diam erat fermentum justo nec condimentum neque sapien placerat ante nulla justo aliquam quis turpis eget elit'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (990, 'Meneur', 'cmeneurrh@uiuc.edu', 'Male', 'Maroon', null, false, 'vestibulum proin eu mi nulla ac enim in tempor turpis nec euismod scelerisque'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (991, 'Sprowles', 'ssprowlesri@virginia.edu', 'Male', 'Fuscia', null, false, 'phasellus id sapien in sapien iaculis congue vivamus metus arcu adipiscing molestie hendrerit at vulputate'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (992, 'McAneny', 'emcanenyrj@latimes.com', 'Female', 'Fuscia', null, true, 'pellentesque volutpat dui maecenas tristique est et tempus semper est quam pharetra magna ac consequat metus sapien ut nunc'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (993, 'Rispin', 'krispinrk@squidoo.com', 'Female', 'Khaki', null, true, 'aenean sit amet justo morbi ut odio cras mi pede malesuada in imperdiet et'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (994, 'Bebis', 'sbebisrl@apple.com', 'Male', 'Turquoise', null, true, 'eu tincidunt in leo maecenas pulvinar lobortis est phasellus sit amet erat nulla tempus vivamus in felis eu'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (995, 'Naylor', 'rnaylorrm@sourceforge.net', 'Male', 'Mauv', null, false, 'turpis enim blandit mi in porttitor pede justo eu massa donec dapibus duis at velit eu est congue elementum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (996, 'Wofenden', 'ewofendenrn@yelp.com', 'Female', 'Khaki', null, false, 'et ultrices posuere cubilia curae mauris viverra diam vitae quam suspendisse potenti nullam porttitor lacus at'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (997, 'Gingle', 'wginglero@whitehouse.gov', 'Male', 'Goldenrod', null, true, 'scelerisque mauris sit amet eros suspendisse accumsan tortor quis turpis sed ante vivamus'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (998, 'Bouttell', 'cbouttellrp@geocities.jp', 'Male', 'Indigo', null, true, 'sollicitudin mi sit amet lobortis sapien sapien non mi integer ac neque duis bibendum'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (999, 'Barracks', 'cbarracksrq@friendfeed.com', 'Female', 'Green', null, false, 'aliquam sit amet diam in magna bibendum imperdiet nullam orci'); +insert into main (id, last_name, email, gender, color, nulls, booleans, words) values (1000, 'Renfree', 'srenfreerr@soup.io', 'Female', 'Red', null, false, 'blandit non interdum in ante vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae duis'); diff --git a/tests/sql/secondary.sql b/tests/sql/secondary.sql new file mode 100644 index 000000000..29fc32bbe --- /dev/null +++ b/tests/sql/secondary.sql @@ -0,0 +1,506 @@ +create table secondary ( + timezone VARCHAR(50), + lat VARCHAR(50), + lng VARCHAR(50), + json VARCHAR(50) +); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -22.0077937, -51.5531899, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 6.4918217, 101.3885001, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Seoul', '34.85028', '128.58861', '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.33731, 108.3258329, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.6411839, 121.0469082, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 39.224791, 117.135488, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -17.9659326, -49.6503106, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 32.047749, 118.803753, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Chisinau', 46.7306768, 29.9803379, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Karachi', 30.1102634, 71.4201301, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Baku', 40.4304275, 50.0347608, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 57.7189182, 13.0953584, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -23.2251621, -45.8861749, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -9.6277897, -74.9749415, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 27.135999, 111.106104, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 61.8099494, 30.9377106, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Johannesburg', -34.0736398, 18.470633, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Australia/Sydney', -37.713929, 144.962411, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Uzhgorod', 49.660696, 25.1434666, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 52.3310677, 20.9706537, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.4066528, 108.7719766, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 26.6732151, -81.8181961, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 38.7519042, -9.4389381, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 14.6014909, 103.0933171, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 23.945891, 104.412274, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.7342524, 121.0702642, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Prague', 50.4252048, 14.2604647, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 38.416663, 112.734174, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 36.684789, 118.479654, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.2022247, -8.5597719, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 38.1600508, 138.3046891, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', '6.95333', '124.42556', '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/London', 52.4608207, -1.8886845, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 15.5835978, 102.4253001, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 44.840524, 82.353656, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -6.9556847, 112.4917291, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.7299584, 121.050452, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tbilisi', 43.1023643, 40.618282, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 31.476202, 92.051239, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.2719005, 112.7196817, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -9.8867238, 124.2477171, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bishkek', 42.5317628, 72.2304571, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 35.87616, 102.801062, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 31.7659313, 120.2951156, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Mexico_City', 20.775698, -103.6951543, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ulaanbaatar', 49.6428895, 100.1771896, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Minsk', 53.6232372, 27.8996001, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Libreville', 1.5991751, 11.5759672, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 38.8987281, -77.0336768, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -34.6126135, -58.5321306, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Mexico_City', 19.4137276, -102.0662939, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 40.9303255, -8.4956553, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 17.5358596, 121.6851715, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Kiev', 51.7754831, 32.2595792, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yekaterinburg', 57.2840834, 55.4543246, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -6.9353405, -76.7711626, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Santiago', -38.5084312, -72.4541343, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 35.86166, 104.195397, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Santo_Domingo', 18.907778, -70.719444, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Nicosia', 35.1338673, 33.0817144, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 36.179383, 112.804103, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 33.7403041, 130.9463113, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Almaty', 55.0449125, 67.8812063, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Ljubljana', 46.2209242, 14.3353423, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Zagreb', 45.7705013, 15.9679074, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.9396646, 111.4109784, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 27.232443, 112.738604, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 58.6073483, 14.0368868, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Pyongyang', '40.38944', '125.04667', '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 22.529403, 103.93935, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.9528113, 111.9880199, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 21.7757592, 106.9594723, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Sakhalin', 43.4554, 43.5944, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Boa_Vista', 2.8235098, -60.6758331, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 16.487885, 121.159718, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Harbin', 25.112046, 99.161761, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Tirane', 40.6098385, 20.6898971, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Berlin', 51.256002, 7.1736304, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Almaty', 49.4129509, 75.4772753, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 59.3814076, 17.9744029, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 52.4368772, 16.826031, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.7870665, -7.9442479, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Los_Angeles', 36.09, -115.15, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.3623241, 106.0991261, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 10.5308066, 105.1609838, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Pacific/Auckland', -35.229035, 173.94869, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Berlin', 48.8096645, 9.2415192, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 13.9619203, 100.5461093, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Toronto', '46.70011', '-75.44929', '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Guatemala', 14.5324564, -91.512542, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 7.216175, 7.5599321, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 36.9746168, 136.9251078, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 4.575601, -75.972669, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 28.846966, 120.728801, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', 0.0925393, 109.2891005, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 56.546084, 13.7445655, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 24.64995, 116.156645, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -8.8196754, 121.0507098, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 26.053583, 117.870803, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 39.042852, 117.318781, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jerusalem', 28.9734441, 34.6534332, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.5608072, -8.5863716, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.1674668, 18.9067921, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 52.6053795, 39.5788904, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 39.4721645, -8.5034561, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -5.7577862, 105.6984455, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Aden', '12.78987', '45.00244', '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -34.7725956, -58.6529983, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 34.56505, 95.661523, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Minsk', 53.9045398, 27.5615244, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 29.079175, 119.64742, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 7.743026, 124.210303, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ashgabat', 41.8639083, 59.6326482, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 54.789029, 39.833505, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 44.3559175, 41.5113076, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 59.3635749, 17.9932873, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.4011293, 22.904714, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 31.7338222, 118.8719394, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 23.028301, 115.84573, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Guayaquil', -1.4493473, -79.4634825, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 59.2255188, 38.5125708, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Los_Angeles', 33.8101772, -118.3520389, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 13.6132538, 100.8282161, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 22.326471, 112.578178, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 49.9727767, 21.5148018, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 57.6995979, 11.9761953, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 44.6962724, 39.8024873, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 23.382145, 109.56259, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -8.577582, 115.136457, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 53.2544872, 14.4927021, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 13.8195176, 121.1729455, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -8.1264039, 111.7018452, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 13.151121, 123.3047135, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 34.014215, 105.298756, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 34.6372754, 134.0105824, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Harbin', 40.003695, 116.103452, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 41.068909, 122.949298, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Oral', 46.9691061, 54.0068191, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', '-10.5676', '121.9306', '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jerusalem', 33.207933, 35.570246, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Pyongyang', 38.235241, 125.750504, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 25.953512, 115.685583, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 40.211932, -8.8417565, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 31.031933, 119.659553, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 57.227222, 29.161111, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 44.6962724, 39.8024873, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 47.3599545, 6.3904955, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Niamey', 14.6489525, 2.1450245, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 16.438508, 103.5060994, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 8.9407376, 4.7821043, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 30.406007, 110.971573, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 6.3833462, -75.5858886, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Vilnius', 55.7195009, 21.9281999, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kolkata', 28.274601, 88.520031, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 14.5520342, 105.1960795, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.632526, 121.103, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 3.914099, -75.479861, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 39.109563, 117.223371, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 42.7021916, 9.4476864, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.9586416, 107.6896073, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 18.178456, 105.7028457, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 40.0480719, 20.7528378, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -11.9417972, -76.9970692, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Berlin', 48.2115037, 11.5131806, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 53.2166678, 19.0555956, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 56.8134441, 13.9368114, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.2169163, -8.6759571, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -23.6012232, -51.6392076, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Indian/Mauritius', -20.2513658, 57.6184076, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 23.047191, 112.465091, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Dublin', 53.3881832, -6.2611953, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 13.9686682, 120.7285858, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 47.407461, -1.18752, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', 34.5911393, 68.9569399, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 39.16352, 23.4901667, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jerusalem', 32.777591, 35.305683, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', '-8.521', '123.5353', '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tehran', 32.9886195, 47.4245319, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Riga', 58.2529231, 22.4850409, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -18.4943258, -40.4657826, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 49.0344853, 2.2178783, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 35.444497, 106.73609, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sarajevo', 45.0818466, 18.3957811, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', 1.05, 118.8, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', 34.34013, 63.75288, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Dhaka', 25.9923398, 89.2847251, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kuching', 4.363324, 118.099562, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', 3.706995, 103.1024948, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', '35.83339', '102.6395', '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Los_Angeles', 38.0434058, -121.2969674, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Helsinki', 60.261145, 20.7891235, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Tunis', 33.888077, 10.0975221, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 34.8643622, 138.315371, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Riyadh', 25.5025179, 49.5999539, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 48.7131111, 5.0953327, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sarajevo', 44.8039803, 19.2621377, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Nicosia', 35.057659, 33.2419422, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 40.570338, 116.554665, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Indian/Comoro', -12.3502374, 44.5278242, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 57.759167, 16.6236443, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 24.880992, 118.71899, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -1.7083922, 103.0817903, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -25.2641605, -50.6378804, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Salta', -34.7061803, -58.3124766, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Salta', -38.848827, -68.119936, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Fortaleza', -3.701499, -40.3496894, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 41.76, -72.69, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Fortaleza', -5.4430889, -35.8716137, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Pacific/Guam', 8.7519471, 150.3392188, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 12.252426, 109.095697, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 29.000347, 101.507294, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Guatemala', 15.0690612, -91.6341489, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 3.251528, -76.229027, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 56.6338771, 47.8109854, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Monterrey', 19.4449916, -99.1082647, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 52.9563074, 31.9693094, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yakutsk', '51.55352', '118.3446', '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 45.697904, 0.964671, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 45.293877, 41.0546109, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sofia', 43.7365869, 27.2601131, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 6.3049636, 7.5626411, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 7.9632405, 123.8751554, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 45.0052663, 41.1199008, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.4041582, 105.8730678, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 40.7046234, 23.652122, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Belgrade', 45.9397564, 19.203958, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 30.037192, 121.154634, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Prague', 49.090211, 16.6038816, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Volgograd', 59.0405545, 52.3557986, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 48.4621258, 7.4888434, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 51.8677116, 16.4856013, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Denver', 40.7177351, -111.8985922, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 30.513598, 110.361715, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Luanda', -11.7714258, 19.8901427, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 40.5854429, -8.6170601, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.6374665, 112.8880068, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Amman', 31.9453666, 35.9283716, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/La_Paz', -19.7809492, -63.1973233, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 15.9799959, 120.1954272, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.0453363, 108.2347863, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.7659245, 111.5259628, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 13.1956813, 108.6877976, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 26.170171, 118.190467, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Addis_Ababa', 11.5505858, 41.4362457, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Guatemala', 14.5482727, -89.6618378, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 6.961969, -75.4190689, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Kampala', 1.973103, 34.0641419, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 34.534829, 109.088297, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Zaporozhye', 50.0688235, 34.7645338, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.9666323, 15.0249601, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Prague', 50.0385516, 14.0470364, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 53.36117, 14.8784944, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Karachi', 31.5555726, 74.3925525, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Indian/Antananarivo', -20.2904186, 44.2999955, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 13.3200851, 101.1015323, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Novokuznetsk', 54.914389, 85.646652, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Douala', 4.5778968, 13.3500652, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Kinshasa', -0.2842862, 20.8850949, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 28.945479, 120.379714, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', '36.69132', '68.79093', '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Libreville', -0.8183228, 12.7250036, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 64.8973466, 45.7651734, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.797767, 111.3262355, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.0281297, 18.1278226, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/La_Paz', -21.2664483, -63.4508107, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 32.391736, 119.435046, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 50.300542, 2.801689, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Cairo', 29.8483192, 31.3368529, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Abidjan', 8.2841689, -7.681943, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.1501831, 121.3167785, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/La_Paz', -17.7357805, -65.1922148, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Berlin', 49.031645, 7.950014, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.3947391, 106.0081819, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 41.234822, 100.482354, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 28.074649, 119.141473, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.2169163, -8.6759571, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 7.450268, 8.6091961, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Nairobi', 0.2827307, 34.7518631, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Douala', 3.7953482, 10.1367166, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.7218793, 113.4785837, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yekaterinburg', 57.6442367, 56.4727932, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 34.329605, 108.708991, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', 32.7778898, 61.6567273, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 40.7038704, -74.0138541, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 33.437877, 113.609286, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.2816099, 108.3771645, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 14.1313261, 122.8277624, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Guatemala', 14.5791223, -91.5850599, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', '-9.68', '124.7403', '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Vilnius', '55.41667', '22.61667', '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Costa_Rica', 9.3607067, -83.7145811, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Vladivostok', 55.804047, 37.2877274, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 48.821908, 2.2615744, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -0.6850063, 100.3273368, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 40.5885408, 23.0279289, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Hebron', 31.7777597, 35.2346035, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/London', 51.3614279, -0.193961, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 32.513883, 118.677372, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Brussels', 49.9672504, 5.8900588, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Aden', 15.8, 43.333333, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -2.7217612, 115.2007728, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Karachi', 24.5552816, 68.9071861, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -8.2103859, 113.4469459, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Harbin', 40.621506, 119.9745295, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -6.3934005, -77.4859711, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 31.038781, 114.5465571, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 48.9318625, 2.4943891, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 18.400092, 109.752583, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 12.8897363, 100.9010416, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.4048381, 106.0140586, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Belgrade', 45.7140258, 20.7713965, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 26.312444, 99.850365, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', 33.0517157, 68.0149609, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Bogota', 10.031243, -74.97611, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Hebron', 31.958519, 35.225893, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 31.9311556, 120.3086809, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sofia', 41.5728211, 23.7282719, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 29.859258, 98.007143, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Karachi', 30.7664515, 70.9293858, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 39.7825, 116.3877778, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Damascus', '35.65599', '35.88871', '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ulaanbaatar', 48.659828, 102.625198, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Seoul', 37.5430686, 126.939386, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Gaborone', -24.6131407, 25.8731407, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 52.3537244, 18.6995274, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/St_Lucia', 13.8211871, -60.9001934, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Mbabane', -26.4820262, 31.3484448, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 65.2670135, 21.7880154, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.175324, 106.635071, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Tucuman', -26.8193949, -65.1435685, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 47.0169394, -68.1430203, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Atlantic/Cape_Verde', 14.8951679, -24.4945636, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -12.5909084, -69.1963141, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 22.916134, 114.1002135, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Johannesburg', -28.1804757, 30.1531568, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bishkek', 41.1694718, 75.8098141, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Damascus', 36.2861677, 37.1097748, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.5504089, 20.418024, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Harbin', 44.906512, 127.60654, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -8.1991972, 113.6353431, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Omsk', '52.6551', '85.09858', '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -1.2088413, 116.8776192, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yekaterinburg', 59.7513574, 60.2055668, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -34.493672, -58.5854528, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -21.9310093, -50.5069723, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Seoul', 36.6826123, 126.8483951, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 38.4581162, -7.7521244, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.7543488, 108.4602464, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 47.0163969, -68.1430029, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -17.8599943, -41.5091004, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yerevan', 39.9999288, 44.5583013, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Prague', 49.3564402, 17.3645179, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 59.4075977, 17.9282806, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Vientiane', 15.8171073, 106.2522143, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Kiev', '50.81667', '27.03333', '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -8.0580651, -79.0552494, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tokyo', 33.8676555, 130.8939207, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 60.5381467, 15.427113, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 44.8568897, 38.6720778, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 38.9079399, -77.0717165, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 58.4073691, 15.5151157, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 56.0629097, 14.1774467, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -8.5181178, 122.8282569, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Mexico_City', 20.6523028, -87.0696273, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 38.5434693, -9.0610972, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 51.14554, 21.7749021, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Managua', 12.0962646, -85.3704606, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 38.7374207, -9.3390593, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Luxembourg', 49.8054141, 6.1555655, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Helsinki', 67.9547288, 23.6829386, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 30.051035, 122.0185, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 26.685936, 118.163788, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 39.765701, 118.412764, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 54.2064439, 17.6677767, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Bangkok', 16.1169002, 102.7020218, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 29.5530941, 118.9353968, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 22.569804, 113.869235, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Krasnoyarsk', 51.3842543, 58.9999439, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', '-6.7597', '111.3245', '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Tashkent', 40.6627728, 68.5446228, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 36.759507, 110.632006, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.62508, 19.3633201, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 47.5173564, 42.1602043, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 21.856289, 103.3413048, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Chisinau', 47.6949791, 28.9654659, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 29.265836, 103.552514, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 57.6817979, 11.9836846, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/New_York', 39.9175385, -75.1136721, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Santo_Domingo', 18.9337202, -70.4047314, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Panama', 8.5524587, -82.6861446, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Casablanca', 34.681962, -1.900155, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Pacific/Auckland', -41.748446, 171.596443, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/London', 52.4608207, -1.8886845, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 9.7652591, 8.8597095, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Dhaka', '23.01472', '89.43936', '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 39.3915625, 21.9245288, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -33.1416441, -64.3600342, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 58.909448, 17.9555203, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sofia', 42.7066497, 23.3067353, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Casablanca', 33.3625159, -4.7303397, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', '-7.1278', '108.2102', '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Athens', 38.4791344, 23.4486395, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', 3.6196125, 98.5089455, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kuala_Lumpur', 5.329576, 103.1369108, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 17.1056791, 120.4557309, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Santarem', -0.6801982, -47.3510778, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Vilnius', 55.0775116, 24.2604924, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 30.47863, 111.459076, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -6.588201, 106.804564, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -25.2929784, -49.2266344, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 59.3781172, 16.5102565, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 40.1238547, -8.2410308, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', -5.456385, 122.612261, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 21.853708, 106.761519, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Monterrey', 19.782027, -99.0598732, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Caracas', 8.3537158, -70.5770632, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Tunis', 36.5072263, 8.7756556, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Budapest', 48.1067292, 20.8193403, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Recife', -7.7427496, -35.2280343, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Zurich', 47.0419512, 8.3140814, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -34.6795558, -58.5577501, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -0.8922447, 119.8504075, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Dhaka', 25.6279123, 88.6331758, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Edmonton', 53.5412083, -113.2957365, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 52.596028, 22.1470031, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Pontianak', '4.5557', '95.71508', '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Dakar', 13.449097, -14.769285, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 27.868393, 113.433712, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -22.4412481, -43.4580314, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Ho_Chi_Minh', 19.992977, 105.8451758, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 38.371402, 117.329949, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.9499222, 110.5304842, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Helsinki', 61.7927812, 24.5525054, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Kampala', 2.5282234, 34.6579985, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Tripoli', 32.2785242, 14.5189214, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 38.0068597, -8.385532, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lusaka', -12.6339242, 28.0537367, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Tunis', 33.4614354, 9.0294708, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 23.460229, 114.010317, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Dushanbe', 38.4613914, 70.7896966, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 57.7372668, 40.4312906, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 47.17532, -1.6901492, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Kampala', 1.2061176, 31.8164031, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 4.7442679, 119.402976, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 52.2680883, 22.5839609, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Ouagadougou', 12.6612116, -0.5683399, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 5.1550917, 8.0247005, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Tripoli', 21.8163242, 17.4307172, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.3940988, -8.0276048, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 31.463475, 120.649341, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 49.9521852, 18.4800457, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -23.006144, -46.8422385, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Harbin', 42.654146, 122.749282, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -25.1169056, -49.5076392, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 40.09864, 119.949545, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Cuiaba', -15.8916202, -52.2618826, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Sao_Paulo', -21.2525138, -48.3256762, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Makassar', '-6.9967', '112.8419', '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 48.0080981, 6.7166698, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -13.1638737, -74.2235641, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Dakar', 13.3041159, -14.3864656, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 30.7016468, 104.0663422, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Warsaw', 50.3142814, 18.9285154, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Aden', 15.016667, 49.35, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Argentina/Cordoba', -32.1748082, -64.1108167, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Gaborone', -21.2001343, 24.8656498, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Chicago', 39.1228935, -95.624767, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 43.1162469, 1.6196332, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Vancouver', 49.2849107, -122.8677562, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kabul', 34.016449, 64.5266446, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Lima', -7.2863903, -78.5753117, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 30.169665, 121.266579, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Jakarta', -7.0117033, 108.3682641, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 32.748741, 103.82319, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 10.7930794, 125.6860723, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Sofia', 41.2624266, 23.3738571, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Manila', 5.990033, 120.953061, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Yerevan', 39.9566253, 44.5924796, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Lisbon', 41.3108613, -8.1338719, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Recife', -7.9908056, -34.841629, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Kolkata', 28.1732341, 68.0179319, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Lagos', 6.8907086, 8.352656, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Fortaleza', -7.0294857, -34.8458603, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Chicago', 42.2685543, -89.0965841, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Chongqing', 34.759314, 108.992603, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Porto-Novo', 6.6644629, 2.1597595, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 22.6229329, 113.2773942, '[{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('America/Havana', 22.4963477, -79.5462943, '[{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Stockholm', 59.3203432, 18.0667845, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Shanghai', 22.396428, 114.109497, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Dublin', 53.32112, -6.2398782, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Paris', 43.4945737, 5.8978018, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Seoul', 36.4465346, 127.1191534, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 45.293877, 41.0546109, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Asia/Hebron', 31.689957, 35.188236, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Prague', 49.7880079, 14.293561, '[{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Moscow', 54.7993525, 38.297354, '[{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Belgrade', 42.0615398, 20.6511592, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Europe/Dublin', 53.3347733, -6.2427004, '[{},{},{},{},{}]'); +insert into secondary (timezone, lat, lng, json) values ('Africa/Ndjamena', '8.6761', '16.566', '[{},{},{},{}]'); diff --git a/tests/test_ml.py b/tests/test_ml.py index 47f4ee112..8cdfb9809 100644 --- a/tests/test_ml.py +++ b/tests/test_ml.py @@ -8,12 +8,10 @@ import optimus.ml.feature as fe -op = Optimus() -spark = op.spark -sc = op.sc +op = Optimus(master='local') -df_cancer = spark.read.csv('tests/data_cancer.csv', sep=',', header=True, inferSchema=True) +df_cancer = op.spark.read.csv('tests/data_cancer.csv', sep=',', header=True, inferSchema=True) columns = ['diagnosis', 'radius_mean', 'texture_mean', 'perimeter_mean', 'area_mean', 'smoothness_mean', 'compactness_mean', 'concavity_mean', 'concave points_mean', 'symmetry_mean', 'fractal_dimension_mean'] @@ -32,7 +30,7 @@ def assert_spark_model(model): def test_ml_pipe(): - df = sc. \ + df = op.sc. \ parallelize([Row(sentence='this is a test', label=0.), Row(sentence='this is another test', label=1.)]). \ toDF() @@ -72,6 +70,24 @@ def test_n_gram(): assert_equal(df_model.select('sentence', 'features').count(), 2) +def test_string_to_index_kargs(): + df = op.spark.createDataFrame([(0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, "a"), (5, "c")], + ["id", "category"]) + + df_indexed = fe.string_to_index(df, "category", stringOrderType="frequencyAsc") + + assert_spark_df(df_indexed) + + expected_collect = op.sc.parallelize([Row(id=0, category='a', category_index=2.0), + Row(id=1, category='b', category_index=0.0), + Row(id=2, category='c', category_index=1.0), + Row(id=3, category='a', category_index=2.0), + Row(id=4, category='a', category_index=2.0), + Row(id=5, category='c', category_index=1.0)]).toDF() + + assert_equal(df_indexed.select("category", "category_index", "id").collect(), expected_collect.collect()) + + def test_random_forest(): df_model, rf_model = op.ml.random_forest(df_cancer, columns, "diagnosis") @@ -101,25 +117,28 @@ def test_h2o_automl(): assert_spark_df(df_model) - assert isinstance(automl_model, py_sparkling.ml.models.H2OAutoMLModel), "Not a H2OAutoMLModel" + assert isinstance(automl_model, py_sparkling.ml.models.H2OMOJOModel), "Not a H2OMOJOModel" + def test_h2o_deeplearning(): df_model, dl_model = op.ml.h2o_deeplearning(df_cancer, "diagnosis", columns_h2o) assert_spark_df(df_model) - assert isinstance(dl_model, py_sparkling.ml.models.H2ODeepLearningModel), "Not a H2ODeepLearningModel" + assert isinstance(dl_model, py_sparkling.ml.models.H2OMOJOModel), "Not a H2OMOJOModel" + def test_h2o_xgboost(): df_model, xgboost_model = op.ml.h2o_xgboost(df_cancer, "diagnosis", columns_h2o) assert_spark_df(df_model) - assert isinstance(xgboost_model, py_sparkling.ml.models.H2OXGBoostModel), "Not a H2OXGBoostModel" + assert isinstance(xgboost_model, py_sparkling.ml.models.H2OMOJOModel), "Not a H2OMOJOModel" + def test_h2o_gbm(): df_model, gbm_model = op.ml.h2o_gbm(df_cancer, "diagnosis", columns_h2o) assert_spark_df(df_model) - assert isinstance(gbm_model, py_sparkling.ml.models.H2OGBMModel), "Not a H2OGBMModel" + assert isinstance(gbm_model, py_sparkling.ml.models.H2OMOJOModel), "Not a H2OMOJOModel"