From f4cb3283a72e0120ef874b3ef1c1ced37ffc4e9a Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Thu, 20 Jul 2017 11:02:50 -0400
Subject: [PATCH 1/6] restore tests, branch now tracking with
 sql_datastore/feat/query_orders branch

---
 collection.go           |  43 +-----------
 collection_item.go      | 148 ++++++++++++++++++++++++++++++++++++++++
 collection_item_test.go | 116 +++++++++++++++++++++++++++++++
 collection_items.go     |  67 ++++++++++++++++++
 datarepo.go             |   4 +-
 link.go                 |   4 +-
 primer.go               |   4 +-
 queries.go              |  63 ++++++++++++++---
 source.go               |   4 +-
 sql/schema.sql          |  12 ++--
 uncrawlable.go          |   4 +-
 url.go                  |   5 +-
 url_test.go             |  74 ++++++++++++++++++++
 13 files changed, 481 insertions(+), 67 deletions(-)
 create mode 100644 collection_item.go
 create mode 100644 collection_item_test.go
 create mode 100644 collection_items.go

diff --git a/collection.go b/collection.go
index 38e39b3..e5948ea 100644
--- a/collection.go
+++ b/collection.go
@@ -2,7 +2,6 @@ package archive
 
 import (
 	"database/sql"
-	"encoding/json"
 	"fmt"
 	"github.com/datatogether/sql_datastore"
 	"github.com/datatogether/sqlutil"
@@ -30,10 +29,6 @@ type Collection struct {
 	Description string `json:"description"`
 	// url this collection originates from
 	Url string `json:"url,omitempty"`
-	// csv column headers, first value must always be "hash"
-	Schema []string `json:"schema,omitempty"`
-	// actuall collection contents
-	Contents [][]string `json:"contents,omitempty"`
 }
 
 func (c Collection) DatastoreType() string {
@@ -90,9 +85,9 @@ func (c *Collection) Delete(store datastore.Datastore) error {
 	return store.Delete(c.Key())
 }
 
-func (c *Collection) NewSQLModel(id string) sql_datastore.Model {
+func (c *Collection) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &Collection{
-		Id: id,
+		Id: key.Name(),
 	}
 }
 
@@ -124,15 +119,6 @@ func (c *Collection) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 	case sql_datastore.CmdList:
 		return nil
 	default:
-		schemaBytes, err := json.Marshal(c.Schema)
-		if err != nil {
-			panic(err)
-		}
-		contentBytes, err := json.Marshal(c.Contents)
-		if err != nil {
-			panic(err)
-		}
-
 		return []interface{}{
 			c.Id,
 			c.Created.In(time.UTC),
@@ -141,8 +127,6 @@ func (c *Collection) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 			c.Title,
 			c.Description,
 			c.Url,
-			schemaBytes,
-			contentBytes,
 		}
 	}
 }
@@ -153,34 +137,15 @@ func (c *Collection) UnmarshalSQL(row sqlutil.Scannable) (err error) {
 	var (
 		id, creator, title, description, url string
 		created, updated                     time.Time
-		schemaBytes, contentBytes            []byte
 	)
 
-	if err := row.Scan(&id, &created, &updated, &creator, &title, &description, &url, &schemaBytes, &contentBytes); err != nil {
+	if err := row.Scan(&id, &created, &updated, &creator, &title, &description, &url); err != nil {
 		if err == sql.ErrNoRows {
 			return ErrNotFound
 		}
 		return err
 	}
 
-	var schema []string
-	if schemaBytes != nil {
-		schema = []string{}
-		err = json.Unmarshal(schemaBytes, &schema)
-		if err != nil {
-			return err
-		}
-	}
-
-	var contents [][]string
-	if contentBytes != nil {
-		contents = [][]string{}
-		err = json.Unmarshal(contentBytes, &contents)
-		if err != nil {
-			return err
-		}
-	}
-
 	*c = Collection{
 		Id:          id,
 		Created:     created.In(time.UTC),
@@ -189,8 +154,6 @@ func (c *Collection) UnmarshalSQL(row sqlutil.Scannable) (err error) {
 		Title:       title,
 		Description: description,
 		Url:         url,
-		Schema:      schema,
-		Contents:    contents,
 	}
 
 	return nil
diff --git a/collection_item.go b/collection_item.go
new file mode 100644
index 0000000..5fd5677
--- /dev/null
+++ b/collection_item.go
@@ -0,0 +1,148 @@
+package archive
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/datatogether/sql_datastore"
+	"github.com/datatogether/sqlutil"
+	"github.com/ipfs/go-datastore"
+)
+
+type CollectionItem struct {
+	// need a reference to the collection Id to be set to distinguish
+	// this item's membership in this particular list
+	collectionId string
+	// Collection Items are Url's at heart
+	Url
+	// this item's natural index in the colleciton
+	Index int
+	// unique description of this item
+	Description string
+}
+
+func (c CollectionItem) DatastoreType() string {
+	return "CollectionItem"
+}
+
+func (c CollectionItem) GetId() string {
+	return c.Id
+}
+
+func (c CollectionItem) Key() datastore.Key {
+	return datastore.NewKey(fmt.Sprintf("%s:%s/%s", c.DatastoreType(), c.collectionId, c.GetId()))
+}
+
+// Read collection from db
+func (c *CollectionItem) Read(store datastore.Datastore) error {
+	ci, err := store.Get(c.Key())
+	if err != nil {
+		return err
+	}
+
+	got, ok := ci.(*CollectionItem)
+	if !ok {
+		return ErrInvalidResponse
+	}
+	*c = *got
+	return nil
+}
+
+// Save a collection
+func (c *CollectionItem) Save(store datastore.Datastore) (err error) {
+	// var exists bool
+
+	// if c.Id != "" {
+	// 	exists, err = store.Has(c.Key())
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// }
+
+	// if !exists {
+	// 	c.Id = uuid.New()
+	// 	c.Created = time.Now().Round(time.Second)
+	// 	c.Updated = c.Created
+	// } else {
+	// 	c.Updated = time.Now().Round(time.Second)
+	// }
+
+	return store.Put(c.Key(), c)
+}
+
+// Delete a collection, should only do for erronious additions
+func (c *CollectionItem) Delete(store datastore.Datastore) error {
+	return store.Delete(c.Key())
+}
+
+func (c *CollectionItem) NewSQLModel(key datastore.Key) sql_datastore.Model {
+	l := key.List()
+	if len(l) == 2 {
+		return &CollectionItem{
+			collectionId: l[0],
+			Url:          Url{Id: l[1]},
+		}
+	}
+	return &CollectionItem{}
+}
+
+func (c CollectionItem) SQLQuery(cmd sql_datastore.Cmd) string {
+	switch cmd {
+	case sql_datastore.CmdCreateTable:
+		return qCollectionItemCreateTable
+	case sql_datastore.CmdExistsOne:
+		return qCollectionItemExists
+	case sql_datastore.CmdSelectOne:
+		return qCollectionItemById
+	case sql_datastore.CmdInsertOne:
+		return qCollectionItemInsert
+	case sql_datastore.CmdUpdateOne:
+		return qCollectionItemUpdate
+	case sql_datastore.CmdDeleteOne:
+		return qCollectionItemDelete
+	case sql_datastore.CmdList:
+		return qCollectionItems
+	default:
+		return ""
+	}
+}
+
+func (c *CollectionItem) SQLParams(cmd sql_datastore.Cmd) []interface{} {
+	switch cmd {
+	case sql_datastore.CmdSelectOne, sql_datastore.CmdExistsOne, sql_datastore.CmdDeleteOne:
+		return []interface{}{c.collectionId, c.Url.Id}
+	case sql_datastore.CmdList:
+		return nil
+	default:
+		return []interface{}{
+			c.collectionId,
+			c.Url.Id,
+			c.Index,
+			c.Description,
+		}
+	}
+}
+
+// UnmarshalSQL reads an sql response into the collection receiver
+// it expects the request to have used collectionCols() for selection
+func (c *CollectionItem) UnmarshalSQL(row sqlutil.Scannable) (err error) {
+	var (
+		collectionId, urlId, description string
+		index                            int
+	)
+
+	if err := row.Scan(&collectionId, &urlId, &index, &description); err != nil {
+		if err == sql.ErrNoRows {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	*c = CollectionItem{
+		collectionId: collectionId,
+		Url:          Url{Id: urlId},
+		Index:        index,
+		Description:  description,
+	}
+
+	return nil
+}
diff --git a/collection_item_test.go b/collection_item_test.go
new file mode 100644
index 0000000..eef0cf7
--- /dev/null
+++ b/collection_item_test.go
@@ -0,0 +1,116 @@
+package archive
+
+import (
+	"fmt"
+	"github.com/datatogether/sql_datastore"
+	"github.com/ipfs/go-datastore"
+	"testing"
+)
+
+// confirm CollectionItem conforms to sql datastore mode
+var _ = sql_datastore.Model(&CollectionItem{})
+
+func TestCollectionItemStorage(t *testing.T) {
+	store := datastore.NewMapDatastore()
+
+	c := &Collection{Title: "test collection"}
+	if err := c.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	u := Url{Url: "http://example.url.test.com"}
+
+	item := &CollectionItem{collectionId: c.Id, Url: u, Index: 0, Description: "Description"}
+	if err := item.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	item2 := &CollectionItem{collectionId: c.Id, Url: u}
+	if err := item2.Read(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if err := CompareCollectionItems(item, item2); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	item.Description = "Updated Description"
+	if err := item.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	item2 = &CollectionItem{collectionId: c.Id, Url: u}
+	if err := item2.Read(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if err := CompareCollectionItems(item, item2); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if err := item.Delete(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+}
+
+func TestCollectionItemSQLStorage(t *testing.T) {
+	// defer resetTestData(appDB, "collections")
+
+	// store := sql_datastore.NewDatastore(appDB)
+	// if err := store.Register(&Collection{}); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+
+	// c := &Collection{Title: "test collection"}
+	// if err := c.Save(store); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+
+	// c.Creator = "penelope"
+	// if err := c.Save(store); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+
+	// c2 := &Collection{Id: c.Id}
+	// if err := c2.Read(store); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+
+	// if err := CompareCollections(c, c2); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+
+	// if err := c.Delete(store); err != nil {
+	// 	t.Error(err.Error())
+	// 	return
+	// }
+}
+
+func CompareCollectionItems(a, b *CollectionItem) error {
+	if err := CompareUrls(&a.Url, &b.Url); err != nil {
+		return fmt.Errorf("url mismatch: %s", err.Error())
+	}
+	if a.collectionId != b.collectionId {
+		return fmt.Errorf("collectionId mismatch: %d != %d", a.Index, b.Index)
+	}
+	if a.Index != b.Index {
+		return fmt.Errorf("Index mismatch: %d != %d", a.Index, b.Index)
+	}
+	if a.Description != b.Description {
+		return fmt.Errorf("Description mistmatch: %s != %s", a.Description, b.Description)
+	}
+	return nil
+}
diff --git a/collection_items.go b/collection_items.go
new file mode 100644
index 0000000..1541432
--- /dev/null
+++ b/collection_items.go
@@ -0,0 +1,67 @@
+package archive
+
+import (
+	"github.com/datatogether/sql_datastore"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/query"
+)
+
+// ItemCount gets the number of items in the list
+// TODO
+func (c *Collection) ItemCount(store datastore.Datastore) (int, error) {
+	return 0, nil
+}
+
+func (c *Collection) SaveItems(store datastore.Datastore, items []*CollectionItem) error {
+	for _, item := range items {
+		item.collectionId = c.Id
+		if err := item.Save(store); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Collection) DeleteItems(store datastore.Datastore, items []*CollectionItem) error {
+	for _, item := range items {
+		item.collectionId = c.Id
+		if err := item.Delete(store); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit, offset int) (items []*CollectionItem, err error) {
+	items = make([]*CollectionItem, limit)
+
+	res, err := store.Query(query.Query{
+		Limit:  limit,
+		Offset: offset,
+		Orders: []query.Order{
+			query.OrderByValue{
+				TypedOrder: sql_datastore.OrderBy(orderby),
+			},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	i := 0
+	for r := range res.Next() {
+		if r.Error != nil {
+			return nil, err
+		}
+
+		c, ok := r.Value.(*CollectionItem)
+		if !ok {
+			return nil, ErrInvalidResponse
+		}
+
+		items[i] = c
+		i++
+	}
+
+	return items[:i], nil
+}
diff --git a/datarepo.go b/datarepo.go
index eb31195..0e6283f 100644
--- a/datarepo.go
+++ b/datarepo.go
@@ -80,9 +80,9 @@ func (d *DataRepo) Delete(store datastore.Datastore) error {
 	return store.Delete(d.Key())
 }
 
-func (d *DataRepo) NewSQLModel(id string) sql_datastore.Model {
+func (d *DataRepo) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &DataRepo{
-		Id: id,
+		Id: key.Name(),
 	}
 }
 
diff --git a/link.go b/link.go
index 80f437f..707b9c2 100644
--- a/link.go
+++ b/link.go
@@ -109,9 +109,9 @@ func (l *Link) calcHash() {
 	l.Hash = hex.EncodeToString(mhBuf)
 }
 
-func (l *Link) NewSQLModel(id string) sql_datastore.Model {
+func (l *Link) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &Link{
-		Hash: id,
+		Hash: key.Name(),
 		Src:  l.Src,
 		Dst:  l.Dst,
 	}
diff --git a/primer.go b/primer.go
index 5b3cb7a..4ae7c45 100644
--- a/primer.go
+++ b/primer.go
@@ -181,8 +181,8 @@ func (p *Primer) Delete(store datastore.Datastore) error {
 	return store.Delete(p.Key())
 }
 
-func (p *Primer) NewSQLModel(id string) sql_datastore.Model {
-	return &Primer{Id: id}
+func (p *Primer) NewSQLModel(key datastore.Key) sql_datastore.Model {
+	return &Primer{Id: key.Name()}
 }
 
 func (p *Primer) SQLQuery(cmd sql_datastore.Cmd) string {
diff --git a/queries.go b/queries.go
index cd16426..f13b1cf 100644
--- a/queries.go
+++ b/queries.go
@@ -8,16 +8,14 @@ CREATE TABLE IF NOT EXISTS collections (
   updated          timestamp NOT NULL,
   creator          text NOT NULL DEFAULT '',
   title            text NOT NULL DEFAULT '',
-  url              text NOT NULL DEFAULT '',
-  schema           json,
-  contents         json
+  url              text NOT NULL DEFAULT ''
 );`
 
 // list collections by reverse cronological date created
 // paginated
 const qCollections = `
 SELECT
-  id, created, updated, creator, title, description, url, schema, contents
+  id, created, updated, creator, title, description, url
 FROM collections 
 ORDER BY created DESC 
 LIMIT $1 OFFSET $2;`
@@ -25,7 +23,7 @@ LIMIT $1 OFFSET $2;`
 // list collections by creator
 const qCollectionsByCreator = `
 SELECT 
-  id, created, updated, creator, title, description, url, schema, contents 
+  id, created, updated, creator, title, description, url 
 FROM collections
 WHERE creator = $4
 ORDER BY $3
@@ -33,25 +31,25 @@ LIMIT $1 OFFSET $2;`
 
 // check for existence of a collection
 const qCollectionExists = `
-  SELECT exists(SELECT 1 FROM collections WHERE id = $1)
+SELECT exists(SELECT 1 FROM collections WHERE id = $1)
 `
 
 // insert a collection
 const qCollectionInsert = `
 INSERT INTO collections 
-  (id, created, updated, creator, title, description, url, schema, contents ) 
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`
+  (id, created, updated, creator, title, description, url ) 
+VALUES ($1, $2, $3, $4, $5, $6, $7);`
 
 // update an existing collection, selecting by ID
 const qCollectionUpdate = `
 UPDATE collections 
-SET created=$2, updated=$3, creator=$4, title=$5, description=$6, url=$7, schema=$8, contents=$9
+SET created=$2, updated=$3, creator=$4, title=$5, description=$6, url=$7
 WHERE id = $1;`
 
 // read collection info by ID
 const qCollectionById = `
 SELECT 
-  id, created, updated, creator, title, description, url, schema, contents 
+  id, created, updated, creator, title, description, url 
 FROM collections 
 WHERE id = $1;`
 
@@ -60,6 +58,51 @@ const qCollectionDelete = `
 DELETE from collections 
 WHERE id = $1;`
 
+const qCollectionItemCreateTable = `
+CREATE TABLE IF NOT EXISTS collection_items (
+  collection_id    UUID NOT NULL,
+  url_id           text NOT NULL default '',
+  index            integer NOT NULL default -1,
+  description      text NOT NULL default '',
+  PRIMARY KEY      (collection_id, url_id)
+);`
+
+const qCollectionItemInsert = `
+INSERT INTO collection_items
+  (collection_id, url_id, index, description)
+VALUES
+  ($1, $2, $3, $4);`
+
+const qCollectionItemUpdate = `
+UPDATE collection_items
+SET index = $3, description = $4
+WHERE collection_id = $1 and id = $2;`
+
+const qCollectionItemDelete = `
+DELETE collection_items 
+WHERE collection_id = $1 and id = $2;`
+
+const qCollectionItemExists = `
+SELECT exists(SELECT 1 FROM collection_items where id = $2);`
+
+const qCollectionItemById = `
+SELECT
+  ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
+FROM collection_items as ci, urls as u
+WHERE collection_id = $1 AND url_id = $2;`
+
+const qCollectionLength = `
+SELECT count(1) FROM collection_items WHERE collection_id = $1 and url_id = $2;`
+
+const qCollectionItems = `
+SELECT
+  ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
+FROM collection_items as ci, urls as u
+WHERE collection_id = $4
+AND u.id = ci.url_id
+ORDER BY $3
+LIMIT $1 OFFSET $2;`
+
 // insert a dataRepo
 const qDataRepoInsert = `
 INSERT INTO data_repos 
diff --git a/source.go b/source.go
index 7680437..793836d 100644
--- a/source.go
+++ b/source.go
@@ -243,9 +243,9 @@ func (s *Source) Delete(store datastore.Datastore) error {
 	return store.Delete(s.Key())
 }
 
-func (s *Source) NewSQLModel(id string) sql_datastore.Model {
+func (s *Source) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &Source{
-		Id:  id,
+		Id:  key.Name(),
 		Url: s.Url,
 	}
 }
diff --git a/sql/schema.sql b/sql/schema.sql
index 909a546..5301b17 100644
--- a/sql/schema.sql
+++ b/sql/schema.sql
@@ -96,11 +96,13 @@ CREATE TABLE IF NOT EXISTS collections (
   contents         json
 );
 
--- name: create-collection_contents
-CREATE TABLE IF NOT EXISTS collection_contents (
-	collection_id    UUID NOT NULL,
-	hash             text NOT NULL default '',
-	PRIMARY KEY      (collection_id, hash)
+-- name: create-collection_items
+CREATE TABLE IF NOT EXISTS collection_items (
+  collection_id    UUID NOT NULL,
+  url_id           text NOT NULL default '',
+  index            integer NOT NULL default -1,
+  description      text NOT NULL default '',
+  PRIMARY KEY      (collection_id, url_id)
 );
 
 -- name: create-uncrawlables
diff --git a/uncrawlable.go b/uncrawlable.go
index ef7bb13..91477a5 100644
--- a/uncrawlable.go
+++ b/uncrawlable.go
@@ -124,9 +124,9 @@ func (u *Uncrawlable) Delete(store datastore.Datastore) error {
 	return store.Delete(u.Key())
 }
 
-func (u *Uncrawlable) NewSQLModel(id string) sql_datastore.Model {
+func (u *Uncrawlable) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &Uncrawlable{
-		Id:  id,
+		Id:  key.Name(),
 		Url: u.Url,
 	}
 }
diff --git a/url.go b/url.go
index 04362fa..0e95ca1 100644
--- a/url.go
+++ b/url.go
@@ -39,6 +39,7 @@ var notContentExtensions = map[string]bool{
 }
 
 // URL represents... a url.
+// TODO - consider renaming to Resource
 type Url struct {
 	// version 4 uuid
 	// urls can/should/must also be be uniquely identified by Url
@@ -497,9 +498,9 @@ func (u *Url) HeadersMap() (headers map[string]string) {
 	return
 }
 
-func (u *Url) NewSQLModel(id string) sql_datastore.Model {
+func (u *Url) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &Url{
-		Id:   id,
+		Id:   key.Name(),
 		Url:  u.Url,
 		Hash: u.Hash,
 	}
diff --git a/url_test.go b/url_test.go
index ad8d7a8..b427b3a 100644
--- a/url_test.go
+++ b/url_test.go
@@ -1,6 +1,7 @@
 package archive
 
 import (
+	"fmt"
 	// "fmt"
 	"github.com/datatogether/sql_datastore"
 	"github.com/ipfs/go-datastore"
@@ -162,3 +163,76 @@ func TestUrlSuspectedContentUrl(t *testing.T) {
 		}
 	}
 }
+
+func CompareUrls(a, b *Url) error {
+	if a == nil && b != nil || a != nil && b == nil {
+		return fmt.Errorf("nil mismatch %s != %s", a, b)
+	} else if a == nil && b == nil {
+		return nil
+	}
+
+	if a.Id != b.Id {
+		return fmt.Errorf("id mismatch: %s != %s", a.Id, b.Id)
+	}
+
+	if !a.Created.Equal(b.Created) {
+		return fmt.Errorf("created mismatch: %s != %s", a.Created, b.Created)
+	}
+
+	if !a.Updated.Equal(b.Updated) {
+		return fmt.Errorf("updated mismatch: %s != %s", a.Updated, b.Updated)
+	}
+
+	if a.Url != b.Url {
+		return fmt.Errorf("url mismatch: %s != %s ", a.Url, b.Url)
+	}
+	if a.Hash != b.Hash {
+		return fmt.Errorf("Hash mistmatch: %s != %s", a.Hash, b.Hash)
+	}
+
+	if a.LastGet != b.LastGet {
+		return fmt.Errorf("LastGet mistmatch: %s != %s", a.LastGet, b.LastGet)
+	}
+	if a.LastHead != b.LastHead {
+		return fmt.Errorf("LastHead mistmatch: %s != %s", a.LastHead, b.LastHead)
+	}
+	if a.Status != b.Status {
+		return fmt.Errorf("Status mistmatch: %s != %s", a.Status, b.Status)
+	}
+	if a.ContentType != b.ContentType {
+		return fmt.Errorf("ContentType mistmatch: %s != %s", a.ContentType, b.ContentType)
+	}
+	if a.ContentSniff != b.ContentSniff {
+		return fmt.Errorf("ContentSniff mistmatch: %s != %s", a.ContentSniff, b.ContentSniff)
+	}
+	if a.ContentLength != b.ContentLength {
+		return fmt.Errorf("ContentLength mistmatch: %s != %s", a.ContentLength, b.ContentLength)
+	}
+	if a.FileName != b.FileName {
+		return fmt.Errorf("FileName mistmatch: %s != %s", a.FileName, b.FileName)
+	}
+	if a.Title != b.Title {
+		return fmt.Errorf("Title mistmatch: %s != %s", a.Title, b.Title)
+	}
+	if a.DownloadTook != b.DownloadTook {
+		return fmt.Errorf("DownloadTook mistmatch: %s != %s", a.DownloadTook, b.DownloadTook)
+	}
+	if a.HeadersTook != b.HeadersTook {
+		return fmt.Errorf("HeadersTook mistmatch: %s != %s", a.HeadersTook, b.HeadersTook)
+	}
+	// TODO - proper comparison
+	// if a.Headers != b.Headers {
+	// 	return fmt.Errorf("Headers mistmatch: %s != %s", a.Headers, b.Headers)
+	// }
+	// if a.Meta != b.Meta {
+	// 	return fmt.Errorf("Meta mistmatch: %s != %s", a.Meta, b.Meta)
+	// }
+	if a.ContentUrl != b.ContentUrl {
+		return fmt.Errorf("ContentUrl mistmatch: %s != %s", a.ContentUrl, b.ContentUrl)
+	}
+	if a.Uncrawlable != b.Uncrawlable {
+		return fmt.Errorf("Uncrawlable mistmatch: %s != %s", a.Uncrawlable, b.Uncrawlable)
+	}
+
+	return nil
+}

From 209ce60fd6d0d7742bb00f7b35203553c81451a3 Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Thu, 20 Jul 2017 14:56:18 -0400
Subject: [PATCH 2/6] crack at tests passing

---
 collection_item.go      | 53 +++++++++++++-----------
 collection_item_test.go | 92 +++++++++++++++++++++++++++--------------
 main_test.go            |  2 +
 metadata.go             |  2 +-
 queries.go              | 18 ++++----
 source.go               |  2 +-
 sql/schema.sql          |  2 +-
 sql/test_data.sql       |  8 ++++
 url.go                  | 66 ++++++++++++++++++++---------
 url_test.go             |  8 ++--
 10 files changed, 160 insertions(+), 93 deletions(-)

diff --git a/collection_item.go b/collection_item.go
index 5fd5677..bd23a43 100644
--- a/collection_item.go
+++ b/collection_item.go
@@ -9,11 +9,11 @@ import (
 )
 
 type CollectionItem struct {
+	// Collection Items are Url's at heart
+	Url
 	// need a reference to the collection Id to be set to distinguish
 	// this item's membership in this particular list
 	collectionId string
-	// Collection Items are Url's at heart
-	Url
 	// this item's natural index in the colleciton
 	Index int
 	// unique description of this item
@@ -29,11 +29,22 @@ func (c CollectionItem) GetId() string {
 }
 
 func (c CollectionItem) Key() datastore.Key {
-	return datastore.NewKey(fmt.Sprintf("%s:%s/%s", c.DatastoreType(), c.collectionId, c.GetId()))
+	return datastore.NewKey(fmt.Sprintf("%s:%s/%s:%s", Collection{}.DatastoreType(), c.collectionId, c.DatastoreType(), c.GetId()))
 }
 
 // Read collection from db
 func (c *CollectionItem) Read(store datastore.Datastore) error {
+	if c.Url.Id == "" && c.Url.Url != "" {
+		if sqls, ok := store.(*sql_datastore.Datastore); ok {
+			row := sqls.DB.QueryRow(qUrlByUrlString, c.Url.Url)
+			prev := &Url{}
+			if err := prev.UnmarshalSQL(row); err == nil {
+				c.Id = prev.Id
+				// exists = true
+			}
+		}
+	}
+
 	ci, err := store.Get(c.Key())
 	if err != nil {
 		return err
@@ -49,23 +60,14 @@ func (c *CollectionItem) Read(store datastore.Datastore) error {
 
 // Save a collection
 func (c *CollectionItem) Save(store datastore.Datastore) (err error) {
-	// var exists bool
-
-	// if c.Id != "" {
-	// 	exists, err = store.Has(c.Key())
-	// 	if err != nil {
-	// 		return err
-	// 	}
-	// }
-
-	// if !exists {
-	// 	c.Id = uuid.New()
-	// 	c.Created = time.Now().Round(time.Second)
-	// 	c.Updated = c.Created
-	// } else {
-	// 	c.Updated = time.Now().Round(time.Second)
-	// }
+	// fmt.Println("pre url save:", c.Url)
+	u := &c.Url
+	if err := u.Save(store); err != nil {
+		return err
+	}
 
+	c.Url = *u
+	// fmt.Println("post url save:", c.Url)
 	return store.Put(c.Key(), c)
 }
 
@@ -78,8 +80,8 @@ func (c *CollectionItem) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	l := key.List()
 	if len(l) == 2 {
 		return &CollectionItem{
-			collectionId: l[0],
-			Url:          Url{Id: l[1]},
+			collectionId: datastore.NamespaceValue(l[0]),
+			Url:          Url{Id: datastore.NamespaceValue(l[1])},
 		}
 	}
 	return &CollectionItem{}
@@ -125,12 +127,13 @@ func (c *CollectionItem) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 // UnmarshalSQL reads an sql response into the collection receiver
 // it expects the request to have used collectionCols() for selection
 func (c *CollectionItem) UnmarshalSQL(row sqlutil.Scannable) (err error) {
+	// ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
 	var (
-		collectionId, urlId, description string
-		index                            int
+		collectionId, urlId, url, hash, title, description string
+		index                                              int
 	)
 
-	if err := row.Scan(&collectionId, &urlId, &index, &description); err != nil {
+	if err := row.Scan(&collectionId, &urlId, &hash, &url, &title, &index, &description); err != nil {
 		if err == sql.ErrNoRows {
 			return ErrNotFound
 		}
@@ -139,7 +142,7 @@ func (c *CollectionItem) UnmarshalSQL(row sqlutil.Scannable) (err error) {
 
 	*c = CollectionItem{
 		collectionId: collectionId,
-		Url:          Url{Id: urlId},
+		Url:          Url{Id: urlId, Hash: hash, Url: url, Title: title},
 		Index:        index,
 		Description:  description,
 	}
diff --git a/collection_item_test.go b/collection_item_test.go
index eef0cf7..87948e6 100644
--- a/collection_item_test.go
+++ b/collection_item_test.go
@@ -11,6 +11,7 @@ import (
 var _ = sql_datastore.Model(&CollectionItem{})
 
 func TestCollectionItemStorage(t *testing.T) {
+	// store := datastore.NewLogDatastore(datastore.NewMapDatastore(), "CollectionItemTest")
 	store := datastore.NewMapDatastore()
 
 	c := &Collection{Title: "test collection"}
@@ -27,7 +28,7 @@ func TestCollectionItemStorage(t *testing.T) {
 		return
 	}
 
-	item2 := &CollectionItem{collectionId: c.Id, Url: u}
+	item2 := &CollectionItem{collectionId: c.Id, Url: item.Url}
 	if err := item2.Read(store); err != nil {
 		t.Error(err.Error())
 		return
@@ -44,7 +45,7 @@ func TestCollectionItemStorage(t *testing.T) {
 		return
 	}
 
-	item2 = &CollectionItem{collectionId: c.Id, Url: u}
+	item2 = &CollectionItem{collectionId: c.Id, Url: item.Url}
 	if err := item2.Read(store); err != nil {
 		t.Error(err.Error())
 		return
@@ -64,45 +65,72 @@ func TestCollectionItemStorage(t *testing.T) {
 func TestCollectionItemSQLStorage(t *testing.T) {
 	// defer resetTestData(appDB, "collections")
 
-	// store := sql_datastore.NewDatastore(appDB)
-	// if err := store.Register(&Collection{}); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	store := sql_datastore.NewDatastore(appDB)
+	if err := store.Register(&Collection{}, &CollectionItem{}, &Url{}); err != nil {
+		t.Error(err.Error())
+		return
+	}
 
-	// c := &Collection{Title: "test collection"}
-	// if err := c.Save(store); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	c := &Collection{Title: "test collection"}
+	if err := c.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
 
-	// c.Creator = "penelope"
-	// if err := c.Save(store); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	item := &CollectionItem{collectionId: c.Id, Url: Url{Url: "http://test.url.two.example.test"}, Index: 0, Description: "item description"}
+	if err := item.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
 
-	// c2 := &Collection{Id: c.Id}
-	// if err := c2.Read(store); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	item.Description = "updated item description"
+	if err := item.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
 
-	// if err := CompareCollections(c, c2); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	item2 := &CollectionItem{collectionId: c.Id, Url: Url{Id: item.Id}}
+	if err := item2.Read(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
 
-	// if err := c.Delete(store); err != nil {
-	// 	t.Error(err.Error())
-	// 	return
-	// }
+	if err := CompareCollectionItems(item, item2); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	urlItem := &CollectionItem{collectionId: c.Id, Url: Url{Url: item.Url.Url}}
+	if err := urlItem.Read(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if err := CompareCollectionItems(item, urlItem); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if err := item.Delete(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
 }
 
 func CompareCollectionItems(a, b *CollectionItem) error {
-	if err := CompareUrls(&a.Url, &b.Url); err != nil {
-		return fmt.Errorf("url mismatch: %s", err.Error())
+	// TODO - can't compare full urls b/c we don't always fill the whole thing out
+	// if err := CompareUrls(&a.Url, &b.Url); err != nil {
+	// 	return fmt.Errorf("url mismatch: %s", err.Error())
+	// }
+
+	if a.Url.Id != b.Url.Id {
+		return fmt.Errorf("url id mismatch %s != %s", a.Url.Id, b.Url.Id)
 	}
+
+	if a.Url.Url != b.Url.Url {
+		return fmt.Errorf("url mismatch %s != %s", a.Url.Url, b.Url.Url)
+	}
+
 	if a.collectionId != b.collectionId {
 		return fmt.Errorf("collectionId mismatch: %d != %d", a.Index, b.Index)
 	}
diff --git a/main_test.go b/main_test.go
index bd186b3..b1ab12b 100644
--- a/main_test.go
+++ b/main_test.go
@@ -44,6 +44,7 @@ func setupTestDatabase() func() {
 		"metadata",
 		"snapshots",
 		"collections",
+		"collection_items",
 		"archive_requests",
 		"uncrawlables",
 		"data_repos"); err != nil {
@@ -70,6 +71,7 @@ func initializeAppSchema(db *sql.DB) (func(), error) {
 		"create-metadata",
 		"create-snapshots",
 		"create-collections",
+		"create-collection_items",
 		"create-archive_requests",
 		"create-uncrawlables",
 		"create-data_repos",
diff --git a/metadata.go b/metadata.go
index 893e67b..3ddcbb3 100644
--- a/metadata.go
+++ b/metadata.go
@@ -208,7 +208,7 @@ func (m *Metadata) Write(db *sql.DB) error {
 
 			// TODO - this is a straight set, should be derived from consensus calculation
 			u.Title = str
-			if err := u.Update(store); err != nil {
+			if err := u.Save(store); err != nil {
 				return
 			}
 		}()
diff --git a/queries.go b/queries.go
index f13b1cf..944cc50 100644
--- a/queries.go
+++ b/queries.go
@@ -76,20 +76,20 @@ VALUES
 const qCollectionItemUpdate = `
 UPDATE collection_items
 SET index = $3, description = $4
-WHERE collection_id = $1 and id = $2;`
+WHERE collection_id = $1 and url_id = $2;`
 
 const qCollectionItemDelete = `
-DELETE collection_items 
-WHERE collection_id = $1 and id = $2;`
+DELETE FROM collection_items 
+WHERE collection_id = $1 AND url_id = $2;`
 
 const qCollectionItemExists = `
-SELECT exists(SELECT 1 FROM collection_items where id = $2);`
+SELECT exists(SELECT 1 FROM collection_items where collection_id = $1 AND url_id = $2);`
 
 const qCollectionItemById = `
 SELECT
-  ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
+  ci.collection_id, u.id, u.hash, u.url, u.title, ci.index, ci.description
 FROM collection_items as ci, urls as u
-WHERE collection_id = $1 AND url_id = $2;`
+WHERE collection_id = $1 AND url_id = $2 AND u.id = ci.url_id;`
 
 const qCollectionLength = `
 SELECT count(1) FROM collection_items WHERE collection_id = $1 and url_id = $2;`
@@ -778,14 +778,14 @@ set
 where id = $1;`
 
 const qUncrawlableByUrl = `
-select 
+SELECT 
   id, url,created,updated,creator_key_id,
   name,email,event_name,agency_name,
   agency_id,subagency_id,org_id,suborg_id,subprimer_id,
   ftp,database,interactive,many_files,
   comments
-from uncrawlables 
-where url = $1;`
+FROM uncrawlables 
+WHERE url = $1;`
 
 const qUncrawlableById = `
 select 
diff --git a/source.go b/source.go
index 793836d..b9a647c 100644
--- a/source.go
+++ b/source.go
@@ -135,7 +135,7 @@ func (c *Source) AsUrl(db *sql.DB) (*Url, error) {
 	u := &Url{Url: addr.String()}
 	if err := u.Read(store); err != nil {
 		if err == ErrNotFound {
-			if err := u.Insert(store); err != nil {
+			if err := u.Save(store); err != nil {
 				return u, err
 			}
 		} else {
diff --git a/sql/schema.sql b/sql/schema.sql
index 5301b17..bbc4b79 100644
--- a/sql/schema.sql
+++ b/sql/schema.sql
@@ -1,5 +1,5 @@
 -- name: drop-all
-DROP TABLE IF EXISTS urls, links, primers, sources, subprimers, alerts, context, metadata, supress_alerts, snapshots, collections, archive_requests, uncrawlables, data_repos;
+DROP TABLE IF EXISTS urls, links, primers, sources, subprimers, alerts, context, metadata, supress_alerts, snapshots, collections, collection_items, archive_requests, uncrawlables, data_repos;
 
 -- name: create-primers
 CREATE TABLE IF NOT EXISTS primers (
diff --git a/sql/test_data.sql b/sql/test_data.sql
index fdc0cd7..1200694 100644
--- a/sql/test_data.sql
+++ b/sql/test_data.sql
@@ -60,6 +60,14 @@ values
 -- name: delete-collections
 delete from collections;
 
+-- name: insert-collection_items
+INSERT into collection_items
+  (collection_id, url_id, index, description)
+VALUES
+  ('76dd07ac-54cb-4f9d-b0a6-88d3d55c0d9d', 'cee7bbd4-2bf9-4b83-b2c8-be6aeb70e771', 0, 'epa url in the collection');
+-- name: delete-collection_items
+DELETE FROM collection_items;
+
 -- name: insert-uncrawlables
 insert into uncrawlables 
   ( id,url,created,updated,creator_key_id,
diff --git a/url.go b/url.go
index 0e95ca1..4a29880 100644
--- a/url.go
+++ b/url.go
@@ -48,9 +48,9 @@ type Url struct {
 	// any normalization. Url strings must always be absolute.
 	Url string `json:"url"`
 	// Created timestamp rounded to seconds in UTC
-	Created time.Time `json:"created"`
+	Created time.Time `json:"created,omitempty"`
 	// Updated timestamp rounded to seconds in UTC
-	Updated time.Time `json:"updated"`
+	Updated time.Time `json:"updated,omitempty"`
 
 	// Timestamp for most recent GET request
 	LastGet *time.Time `json:"lastGet,omitempty"`
@@ -213,7 +213,7 @@ func (u *Url) HandleGetResponse(db *sql.DB, res *http.Response, done func(err er
 		}
 	}
 
-	err = u.Update(store)
+	err = u.Save(store)
 	if err != nil {
 		return
 	}
@@ -386,33 +386,59 @@ func (u *Url) Read(store datastore.Datastore) error {
 	return ErrNotFound
 }
 
-// Insert (create)
-func (u *Url) Insert(store datastore.Datastore) error {
-	u.Created = time.Now().Round(time.Second)
-	u.Updated = u.Created
-	u.Id = uuid.New()
-	return store.Put(u.Key(), u)
-}
+func (u *Url) Save(store datastore.Datastore) (err error) {
+	var exists bool
 
-// Update url db entry
-func (u *Url) Update(store datastore.Datastore) error {
-	// Need to fetch ID
-	if u.Url != "" && u.Id == "" {
-		prev := &Url{Url: u.Url}
-		if err := prev.Read(store); err != ErrNotFound {
+	if u.Id != "" {
+		exists, err = store.Has(u.Key())
+		if err != nil {
 			return err
 		}
-		u.Id = prev.Id
+	} else if sqls, ok := store.(*sql_datastore.Datastore); ok {
+		// if no Id is set, attempt to set one
+		if u.Url != "" {
+			row := sqls.DB.QueryRow(qUrlByUrlString, u.Url)
+			prev := &Url{}
+			if err := prev.UnmarshalSQL(row); err == nil {
+				u.Id = prev.Id
+				exists = true
+			}
+		}
+	}
+
+	// TODO - support fetching ID via url entry
+	// 	// Need to fetch ID
+	// 	if u.Url != "" && u.Id == "" {
+	// 		prev := &Url{Url: u.Url}
+	// 		if err := prev.Read(store); err != ErrNotFound {
+	// 			return err
+	// 		}
+	// 		u.Id = prev.Id
+	// 	}
+
+	if err = u.validate(); err != nil {
+		return
 	}
 
-	u.Updated = time.Now().Round(time.Second)
+	if !exists {
+		u.Id = uuid.New()
+		u.Created = time.Now().Round(time.Second).In(time.UTC)
+		u.Updated = u.Created
+	} else {
+		u.Updated = time.Now().Round(time.Second).In(time.UTC)
+	}
+
+	return store.Put(u.Key(), u)
+}
+
+func (u *Url) validate() error {
 	if u.ContentLength < -1 {
 		u.ContentLength = -1
 	}
 	if u.Status < -1 {
 		u.Status = -1
 	}
-	return store.Put(u.Key(), u)
+	return nil
 }
 
 // Delete a url, should only do for erronious additions
@@ -449,7 +475,7 @@ func (u *Url) ExtractDocLinks(db *sql.DB, doc *goquery.Document) ([]*Link, error
 		// Check to see if url exists, creating if not
 		if err = dst.Read(store); err != nil {
 			if err == ErrNotFound {
-				if err = dst.Insert(store); err != nil {
+				if err = dst.Save(store); err != nil {
 					return
 				}
 			} else {
diff --git a/url_test.go b/url_test.go
index b427b3a..eb2cb8f 100644
--- a/url_test.go
+++ b/url_test.go
@@ -12,13 +12,13 @@ func TestUrlStorage(t *testing.T) {
 	store := datastore.NewMapDatastore()
 
 	u := &Url{Url: "http://youtube.com"}
-	if err := u.Insert(store); err != nil {
+	if err := u.Save(store); err != nil {
 		t.Error(err.Error())
 		return
 	}
 
 	u.ContentLength = 10
-	if err := u.Update(store); err != nil {
+	if err := u.Save(store); err != nil {
 		t.Error(err.Error())
 		return
 	}
@@ -53,13 +53,13 @@ func TestUrlSQLStorage(t *testing.T) {
 	}
 
 	u := &Url{Url: "http://youtube.com"}
-	if err := u.Insert(store); err != nil {
+	if err := u.Save(store); err != nil {
 		t.Error(err.Error())
 		return
 	}
 
 	u.ContentLength = 10
-	if err := u.Update(store); err != nil {
+	if err := u.Save(store); err != nil {
 		t.Error(err.Error())
 		return
 	}

From 38141aa140168a90135c99c6891ad95e8d9db5a6 Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Thu, 20 Jul 2017 17:04:45 -0400
Subject: [PATCH 3/6] added Passing CollectionItems tests

---
 collection_item.go       | 12 ++++---
 collection_items.go      | 37 +++++++++++++++++---
 collection_items_test.go | 75 ++++++++++++++++++++++++++++++++++++++++
 queries.go               |  4 +--
 4 files changed, 117 insertions(+), 11 deletions(-)
 create mode 100644 collection_items_test.go

diff --git a/collection_item.go b/collection_item.go
index bd23a43..a96f6bb 100644
--- a/collection_item.go
+++ b/collection_item.go
@@ -14,7 +14,7 @@ type CollectionItem struct {
 	// need a reference to the collection Id to be set to distinguish
 	// this item's membership in this particular list
 	collectionId string
-	// this item's natural index in the colleciton
+	// this item's index in the collection
 	Index int
 	// unique description of this item
 	Description string
@@ -60,14 +60,12 @@ func (c *CollectionItem) Read(store datastore.Datastore) error {
 
 // Save a collection
 func (c *CollectionItem) Save(store datastore.Datastore) (err error) {
-	// fmt.Println("pre url save:", c.Url)
 	u := &c.Url
 	if err := u.Save(store); err != nil {
 		return err
 	}
 
 	c.Url = *u
-	// fmt.Println("post url save:", c.Url)
 	return store.Put(c.Key(), c)
 }
 
@@ -78,7 +76,11 @@ func (c *CollectionItem) Delete(store datastore.Datastore) error {
 
 func (c *CollectionItem) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	l := key.List()
-	if len(l) == 2 {
+	if len(l) == 1 {
+		return &CollectionItem{
+			collectionId: datastore.NamespaceValue(l[0]),
+		}
+	} else if len(l) == 2 {
 		return &CollectionItem{
 			collectionId: datastore.NamespaceValue(l[0]),
 			Url:          Url{Id: datastore.NamespaceValue(l[1])},
@@ -113,7 +115,7 @@ func (c *CollectionItem) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 	case sql_datastore.CmdSelectOne, sql_datastore.CmdExistsOne, sql_datastore.CmdDeleteOne:
 		return []interface{}{c.collectionId, c.Url.Id}
 	case sql_datastore.CmdList:
-		return nil
+		return []interface{}{c.collectionId}
 	default:
 		return []interface{}{
 			c.collectionId,
diff --git a/collection_items.go b/collection_items.go
index 1541432..b689745 100644
--- a/collection_items.go
+++ b/collection_items.go
@@ -7,9 +7,32 @@ import (
 )
 
 // ItemCount gets the number of items in the list
-// TODO
-func (c *Collection) ItemCount(store datastore.Datastore) (int, error) {
-	return 0, nil
+func (c *Collection) ItemCount(store datastore.Datastore) (count int, err error) {
+	if sqls, ok := store.(*sql_datastore.Datastore); ok {
+		row := sqls.DB.QueryRow(qCollectionLength, c.Id)
+		err = row.Scan(&count)
+		return
+	}
+
+	// TODO - untested code :/
+	res, err := store.Query(query.Query{
+		Prefix:   c.Key().String(),
+		KeysOnly: true,
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	for r := range res.Next() {
+		if r.Error != nil {
+			return 0, err
+		}
+		if _, ok := r.Value.(*CollectionItem); ok {
+			count++
+		}
+	}
+
+	return
 }
 
 func (c *Collection) SaveItems(store datastore.Datastore, items []*CollectionItem) error {
@@ -38,6 +61,10 @@ func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit,
 	res, err := store.Query(query.Query{
 		Limit:  limit,
 		Offset: offset,
+		Prefix: c.Key().String(),
+		Filters: []query.Filter{
+			sql_datastore.FilterKeyTypeEq(CollectionItem{}.DatastoreType()),
+		},
 		Orders: []query.Order{
 			query.OrderByValue{
 				TypedOrder: sql_datastore.OrderBy(orderby),
@@ -51,7 +78,7 @@ func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit,
 	i := 0
 	for r := range res.Next() {
 		if r.Error != nil {
-			return nil, err
+			return nil, r.Error
 		}
 
 		c, ok := r.Value.(*CollectionItem)
@@ -63,5 +90,7 @@ func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit,
 		i++
 	}
 
+	// fmt.Println(items)
+
 	return items[:i], nil
 }
diff --git a/collection_items_test.go b/collection_items_test.go
new file mode 100644
index 0000000..7575375
--- /dev/null
+++ b/collection_items_test.go
@@ -0,0 +1,75 @@
+package archive
+
+import (
+	"github.com/datatogether/sql_datastore"
+	"testing"
+)
+
+func TestCollectionItemsSQLStorage(t *testing.T) {
+	defer resetTestData(appDB, "collection_items", "collections", "urls")
+
+	store := sql_datastore.NewDatastore(appDB)
+	if err := store.Register(&Collection{}, &CollectionItem{}, &Url{}); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	c := &Collection{Title: "test collection Item Count"}
+	if err := c.Save(store); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	count, err := c.ItemCount(store)
+	if err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if count != 0 {
+		t.Errorf("count mistmatch. expected %d, got %d", 0, count)
+		return
+	}
+
+	saveitems := []*CollectionItem{
+		&CollectionItem{Url: Url{Url: "http://test.0.com"}, Index: 0, Description: "zero"},
+		&CollectionItem{Url: Url{Url: "http://test.1.com"}, Index: 1, Description: "one"},
+		&CollectionItem{Url: Url{Url: "http://test.2.com"}, Index: 2, Description: "two"},
+	}
+
+	if err := c.SaveItems(store, saveitems); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if count, err := c.ItemCount(store); err != nil {
+		t.Error(err.Error())
+		return
+	} else if count != len(saveitems) {
+		t.Errorf("count mistmatch. expected %d, got %d", len(saveitems), count)
+		return
+	}
+
+	readitems, err := c.ReadItems(store, "created DESC", 100, 0)
+	if err != nil {
+		t.Error(err.Error())
+		return
+	} else if len(readitems) != len(saveitems) {
+		t.Errorf("count mistmatch. expected %d, got %d", len(saveitems), count)
+		return
+	}
+
+	if err := c.DeleteItems(store, saveitems); err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	if count, err := c.ItemCount(store); err != nil {
+		t.Error(err.Error())
+		return
+	} else if count != 0 {
+		t.Errorf("count mistmatch. expected %d, got %d", 0, count)
+		return
+	}
+
+}
diff --git a/queries.go b/queries.go
index 944cc50..87c4864 100644
--- a/queries.go
+++ b/queries.go
@@ -92,11 +92,11 @@ FROM collection_items as ci, urls as u
 WHERE collection_id = $1 AND url_id = $2 AND u.id = ci.url_id;`
 
 const qCollectionLength = `
-SELECT count(1) FROM collection_items WHERE collection_id = $1 and url_id = $2;`
+SELECT count(1) FROM collection_items WHERE collection_id = $1;`
 
 const qCollectionItems = `
 SELECT
-  ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
+  ci.collection_id, u.id, u.hash, u.url, u.title, ci.index, ci.description
 FROM collection_items as ci, urls as u
 WHERE collection_id = $4
 AND u.id = ci.url_id

From ef12efb511b097baaebe5dbc1458d769d1e42a63 Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Fri, 21 Jul 2017 09:47:42 -0400
Subject: [PATCH 4/6] add documentation

---
 collection_item.go  | 21 +++++++++++++++++----
 collection_items.go | 16 +++++++++++++---
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/collection_item.go b/collection_item.go
index a96f6bb..8b00bd5 100644
--- a/collection_item.go
+++ b/collection_item.go
@@ -8,6 +8,10 @@ import (
 	"github.com/ipfs/go-datastore"
 )
 
+// CollectionItem is an item in a collection. They are urls
+// with added collection-specific information.
+// This has the effect of storing all of the "main properties"
+// of a collection item in the common list of urls
 type CollectionItem struct {
 	// Collection Items are Url's at heart
 	Url
@@ -20,14 +24,20 @@ type CollectionItem struct {
 	Description string
 }
 
+// DatastoreType is to satisfy sql_datastore.Model interface
 func (c CollectionItem) DatastoreType() string {
 	return "CollectionItem"
 }
 
+// GetId returns the Id of the collectionItem, which is the id
+// of the underlying Url
 func (c CollectionItem) GetId() string {
-	return c.Id
+	return c.Url.Id
 }
 
+// Key is somewhat special as CollectionItems always have a Collection
+// as their parent. This relationship is represented in directory-form:
+// /Collection:[collection-id]/CollectionItem:[item-id]
 func (c CollectionItem) Key() datastore.Key {
 	return datastore.NewKey(fmt.Sprintf("%s:%s/%s:%s", Collection{}.DatastoreType(), c.collectionId, c.DatastoreType(), c.GetId()))
 }
@@ -58,7 +68,7 @@ func (c *CollectionItem) Read(store datastore.Datastore) error {
 	return nil
 }
 
-// Save a collection
+// Save a collection item to a store
 func (c *CollectionItem) Save(store datastore.Datastore) (err error) {
 	u := &c.Url
 	if err := u.Save(store); err != nil {
@@ -69,7 +79,7 @@ func (c *CollectionItem) Save(store datastore.Datastore) (err error) {
 	return store.Put(c.Key(), c)
 }
 
-// Delete a collection, should only do for erronious additions
+// Delete a collection item
 func (c *CollectionItem) Delete(store datastore.Datastore) error {
 	return store.Delete(c.Key())
 }
@@ -89,6 +99,8 @@ func (c *CollectionItem) NewSQLModel(key datastore.Key) sql_datastore.Model {
 	return &CollectionItem{}
 }
 
+// SQLQuery is to satisfy the sql_datastore.Model interface, it
+// returns the concrete query for a given type of SQL command
 func (c CollectionItem) SQLQuery(cmd sql_datastore.Cmd) string {
 	switch cmd {
 	case sql_datastore.CmdCreateTable:
@@ -110,6 +122,8 @@ func (c CollectionItem) SQLQuery(cmd sql_datastore.Cmd) string {
 	}
 }
 
+// SQLQuery is to satisfy the sql_datastore.Model interface, it
+// returns this CollectionItem's parameters for a given type of SQL command
 func (c *CollectionItem) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 	switch cmd {
 	case sql_datastore.CmdSelectOne, sql_datastore.CmdExistsOne, sql_datastore.CmdDeleteOne:
@@ -129,7 +143,6 @@ func (c *CollectionItem) SQLParams(cmd sql_datastore.Cmd) []interface{} {
 // UnmarshalSQL reads an sql response into the collection receiver
 // it expects the request to have used collectionCols() for selection
 func (c *CollectionItem) UnmarshalSQL(row sqlutil.Scannable) (err error) {
-	// ci.collection_id, u.id, u.url, u.title, ci.index, ci.description
 	var (
 		collectionId, urlId, url, hash, title, description string
 		index                                              int
diff --git a/collection_items.go b/collection_items.go
index b689745..188627d 100644
--- a/collection_items.go
+++ b/collection_items.go
@@ -6,7 +6,7 @@ import (
 	"github.com/ipfs/go-datastore/query"
 )
 
-// ItemCount gets the number of items in the list
+// ItemCount gets the number of items in the collection
 func (c *Collection) ItemCount(store datastore.Datastore) (count int, err error) {
 	if sqls, ok := store.(*sql_datastore.Datastore); ok {
 		row := sqls.DB.QueryRow(qCollectionLength, c.Id)
@@ -14,7 +14,7 @@ func (c *Collection) ItemCount(store datastore.Datastore) (count int, err error)
 		return
 	}
 
-	// TODO - untested code :/
+	// TODO - untested code :(
 	res, err := store.Query(query.Query{
 		Prefix:   c.Key().String(),
 		KeysOnly: true,
@@ -35,6 +35,9 @@ func (c *Collection) ItemCount(store datastore.Datastore) (count int, err error)
 	return
 }
 
+// SaveItems saves a slice of items to the collection.
+// It's up to you to ensure that the "index" param doesn't get all messed up.
+// TODO - validate / automate the Index param?
 func (c *Collection) SaveItems(store datastore.Datastore, items []*CollectionItem) error {
 	for _, item := range items {
 		item.collectionId = c.Id
@@ -45,6 +48,7 @@ func (c *Collection) SaveItems(store datastore.Datastore, items []*CollectionIte
 	return nil
 }
 
+// DeleteItems removes a given list of items from the collection
 func (c *Collection) DeleteItems(store datastore.Datastore, items []*CollectionItem) error {
 	for _, item := range items {
 		item.collectionId = c.Id
@@ -55,14 +59,21 @@ func (c *Collection) DeleteItems(store datastore.Datastore, items []*CollectionI
 	return nil
 }
 
+// ReadItems reads a bounded set of items from the collection
+// the orderby param currently only supports SQL-style input of a single proprty, eg: "index" or "index DESC"
 func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit, offset int) (items []*CollectionItem, err error) {
 	items = make([]*CollectionItem, limit)
 
 	res, err := store.Query(query.Query{
 		Limit:  limit,
 		Offset: offset,
+		// Keeping in mind that CollectionItem keys take the form /Collection:[id]/CollectionItem:[id]
+		// and Collections have the key /Collection:[id], the Collection key is the prefix for looking up keys
 		Prefix: c.Key().String(),
 		Filters: []query.Filter{
+			// Pass in a Filter Type to specify that results must be of type CollectionItem
+			// In abstract terms this combined with the Prefix query param amounts to querying:
+			// /Collection:[id]/CollectionItem:*
 			sql_datastore.FilterKeyTypeEq(CollectionItem{}.DatastoreType()),
 		},
 		Orders: []query.Order{
@@ -91,6 +102,5 @@ func (c *Collection) ReadItems(store datastore.Datastore, orderby string, limit,
 	}
 
 	// fmt.Println(items)
-
 	return items[:i], nil
 }

From 94bd75820c07fbe22ed2c2e476aee095d1a1eb63 Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Mon, 24 Jul 2017 19:28:33 -0400
Subject: [PATCH 5/6] correct json output of CollectionItem

---
 collection_item.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/collection_item.go b/collection_item.go
index 8b00bd5..26b113a 100644
--- a/collection_item.go
+++ b/collection_item.go
@@ -19,9 +19,9 @@ type CollectionItem struct {
 	// this item's membership in this particular list
 	collectionId string
 	// this item's index in the collection
-	Index int
+	Index int `json:"index"`
 	// unique description of this item
-	Description string
+	Description string `json:"description"`
 }
 
 // DatastoreType is to satisfy sql_datastore.Model interface

From ebb75f8bc4b802393df867ec9cd086a678209d66 Mon Sep 17 00:00:00 2001
From: Brendan O'Brien <sparkle_pony_2000@qri.io>
Date: Tue, 25 Jul 2017 17:32:14 -0400
Subject: [PATCH 6/6] update test data to add collection items

---
 collections_test.go | 10 +++++-----
 sql/test_data.sql   | 38 +++++++++++++++++++++++++++++++-------
 urls_test.go        |  2 +-
 3 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/collections_test.go b/collections_test.go
index 16cbf50..c2caba1 100644
--- a/collections_test.go
+++ b/collections_test.go
@@ -16,7 +16,7 @@ func TestListCollections(t *testing.T) {
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	if len(collections) != 1 {
+	if len(collections) != 4 {
 		t.Errorf("collections length mismatch")
 	}
 
@@ -24,7 +24,7 @@ func TestListCollections(t *testing.T) {
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	if len(collections) != 0 {
+	if len(collections) != 3 {
 		t.Errorf("collections length mismatch")
 	}
 }
@@ -36,11 +36,11 @@ func TestCollectionsByCreator(t *testing.T) {
 		return
 	}
 
-	collections, err := CollectionsByCreator(store, "test_user_key", "created DESC", 20, 0)
+	collections, err := CollectionsByCreator(store, "EDGI_644b51b9567d0d999e40f697d7406a26030cde95a83775d285ff1f57a73b3ebc", "created DESC", 20, 0)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	if len(collections) != 1 {
-		t.Errorf("collections length mismatch")
+	if len(collections) != 2 {
+		t.Errorf("collections length mismatch: %d != %d", len(collections), 2)
 	}
 }
diff --git a/sql/test_data.sql b/sql/test_data.sql
index 1200694..3cf89ab 100644
--- a/sql/test_data.sql
+++ b/sql/test_data.sql
@@ -27,9 +27,15 @@ delete from sources;
 insert into urls
   (url,created,updated,last_head,last_get,status,content_type,content_sniff,content_length,file_name,title,id,headers_took,download_took,headers,meta,hash)
 values
-	('http://www.epa.gov', '2017-01-01 00:00:01', '2017-01-01 00:00:01', '2017-01-01 00:00:01', null, 200, 'text/html; charset=utf-8', 'text/html;', -1, '', 'United States Environmental Protection Agency, US EPA', 'cee7bbd4-2bf9-4b83-b2c8-be6aeb70e771',0,0, '["X-Content-Type-Options","nosniff","Expires","Fri, 24 Feb 2017 21:53:45 GMT","Date","Fri, 24 Feb 2017 21:53:45 GMT","Etag","W/\"7f53-549471782bb42\"","X-Ua-Compatible","IE=Edge,chrome=1","X-Cached-By","Boost","Content-Type","text/html; charset=utf-8","Vary","Accept-Encoding","Accept-Ranges","bytes","Cache-Control","no-cache, no-store, must-revalidate, post-check=0, pre-check=0","Server","Apache","Connection","keep-alive","Strict-Transport-Security","max-age=31536000; preload;"]', null, '1220459219b10032cc86dcdbc0f83aea15a9d3e1119e7b5170beaee233008ea2c2de'),
+  ('http://www.epa.gov', '2017-01-01 00:00:01', '2017-01-01 00:00:01', '2017-01-01 00:00:01', null, 200, 'text/html; charset=utf-8', 'text/html;', -1, '', 'United States Environmental Protection Agency, US EPA', 'cee7bbd4-2bf9-4b83-b2c8-be6aeb70e771',0,0, '["X-Content-Type-Options","nosniff","Expires","Fri, 24 Feb 2017 21:53:45 GMT","Date","Fri, 24 Feb 2017 21:53:45 GMT","Etag","W/\"7f53-549471782bb42\"","X-Ua-Compatible","IE=Edge,chrome=1","X-Cached-By","Boost","Content-Type","text/html; charset=utf-8","Vary","Accept-Encoding","Accept-Ranges","bytes","Cache-Control","no-cache, no-store, must-revalidate, post-check=0, pre-check=0","Server","Apache","Connection","keep-alive","Strict-Transport-Security","max-age=31536000; preload;"]', null, '1220459219b10032cc86dcdbc0f83aea15a9d3e1119e7b5170beaee233008ea2c2de'),
   ('https://www.census.gov/nometa.pdf','2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88', '2017-03-21 22:25:20.88', 200,'text/html','application/pdf; charset=utf-8' ,164010, 'nometa.pdf','North American Industry Classification System (NAICS) Main Page � U.S. Census Bureau','4c5fc7b8-1397-4d34-980b-1d01247f9ee4',0,0 ,'["Date","Tue, 21 Mar 2017 22:25:20 GMT","Accept-Ranges","bytes","Content-Type","text/html","Strict-Transport-Security","max-age=31536000","Vary","Accept-Encoding"]',null,'1220af06510193276b5fd9ad2fc55dcc004ada557d9259ca3505478bfef0b12ed988'),
-  ('https://www.census.gov/topics/economy/classification-codes.html','2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384',200,'text/html','text/plain; charset=utf-8' ,164010, '','North American Industry Classification System (NAICS) Main Page � U.S. Census Bureau','4c5fc7b8-1397-4d34-980b-1d01247f9ee4',0,0 ,'["Date","Tue, 21 Mar 2017 22:25:20 GMT","Accept-Ranges","bytes","Content-Type","text/html","Strict-Transport-Security","max-age=31536000","Vary","Accept-Encoding"]',null,'12207b06510193276b5fd9ad2fc55dcc004ada557d9259ca3505478bfef0b16ed977');
+  ('https://www.census.gov/topics/economy/classification-codes.html','2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384',200,'text/html','text/plain; charset=utf-8' ,164010, '','North American Industry Classification System (NAICS) Main Page � U.S. Census Bureau','4c5fc7b8-1397-4d34-980b-1d01247f9ee4',0,0 ,'["Date","Tue, 21 Mar 2017 22:25:20 GMT","Accept-Ranges","bytes","Content-Type","text/html","Strict-Transport-Security","max-age=31536000","Vary","Accept-Encoding"]',null,'12207b06510193276b5fd9ad2fc55dcc004ada557d9259ca3505478bfef0b16ed977'),
+  ('https://i.imgur.com/LJf4LzX.jpg', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'image/jpeg', 'image/jpeg', 0, '','Puppies!','fe6d9fbd-32fe-4cf3-b48f-8f5010207f4c',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,''),
+  ('https://i.imgur.com/UE4nxKJ.gifv', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'video/mp4', 'video/mp4', 0, '','Puppies!','e1a4eaff-1faf-48ea-9c2c-31968abc82bc',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,''),
+  ('https://i.imgur.com/ku6IEJf.gifv', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'video/mp4', 'video/mp4', 0, '','Puppies!','8596e6b9-9bf6-45d6-b0c2-06a0f71de2df',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,''),
+  ('https://i.imgur.com/y22NjCp.jpg', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'image/jpeg', 'image/jpeg', 0, '','Puppies!','98179ab7-8cd9-4d05-a6c8-24df846b8dd2',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,''),
+  ('https://i.imgur.com/1x8lR0p.jpg', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'image/jpeg', 'image/jpeg', 0, '','Puppies!','41471973-9595-470f-b299-30a3423e267e',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,''),
+  ('http://i.imgur.com/26fVJAE.gifv', '2017-03-15 17:36:40', '2017-03-21 22:25:21','2017-03-21 22:25:20.88','2017-03-21 22:25:20.88384', 200,'video/mp4', 'video/mp4', 0, '','Puppies!','53d4c4bd-9802-41ad-82ae-1e82734d56fc',0,0, '["Date","Tue, 21 Mar 2017 22:25:20 GMT"]',null,'');
 -- name: delete-urls
 delete from urls;
 
@@ -41,7 +47,7 @@ delete from links;
 
 -- name: insert-snapshots
 -- insert into snapshots values
--- 	();
+--  ();
 -- name: delete-snapshots
 delete from snapshots;
 
@@ -54,17 +60,35 @@ delete from metadata;
 
 -- name: insert-collections
 insert into collections 
-  (id, created, updated, creator, title, description, url, schema, contents)
+  (id, created, updated, 
+    creator,
+    title, description, url)
 values
-  ('76dd07ac-54cb-4f9d-b0a6-88d3d55c0d9d', '2017-01-01 00:00:01', '2017-01-01 00:00:01', 'test_user_key', 'Test Collection', 'a collection of urls', '', null, null);
+  ('6995febc-b7be-49ba-8297-1db68a703c3c','2017-07-13 15:57:32','2017-07-13 21:39:38',
+    'EDGI_644b51b9567d0d999e40f697d7406a26030cde95a83775d285ff1f57a73b3ebc',
+    'EPA TRU Datasets', 'essential TRU datasets', ''),
+  ('f444f782-2110-43ca-956c-2c5f0dd56b1a','2017-07-12 23:18:59','2017-07-13 21:57:11',
+    'EDGI_644b51b9567d0d999e40f697d7406a26030cde95a83775d285ff1f57a73b3ebc',
+    'NOAA Volatile Organic Compound CSV files', 'VOC datasests', ''),
+  ('a73a9d04-0fdb-40c8-a97f-288af36e8f6f','2017-07-13 18:45:03','2017-07-13 21:59:25',
+    'blackglade_644b51b9567d0d999e40f697d7406a26030cde95a83775d285ff1f57a73b3ebc',
+    'Test Collection', '', ''),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb','2017-07-11 17:50:19','2017-07-14 00:30:10',
+    'jeffliu_644b51b9567d0d999e40f697d7406a26030cde95a83775d285ff1f57a73b3ebc',
+    'All of the puppies', 'My fav puppy images', '');
 -- name: delete-collections
 delete from collections;
 
 -- name: insert-collection_items
-INSERT into collection_items
+INSERT INTO collection_items
   (collection_id, url_id, index, description)
 VALUES
-  ('76dd07ac-54cb-4f9d-b0a6-88d3d55c0d9d', 'cee7bbd4-2bf9-4b83-b2c8-be6aeb70e771', 0, 'epa url in the collection');
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', 'fe6d9fbd-32fe-4cf3-b48f-8f5010207f4c', 0, 'puppies in a car'),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', 'e1a4eaff-1faf-48ea-9c2c-31968abc82bc', 1, 'puppy in a sweater in a car'),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', '8596e6b9-9bf6-45d6-b0c2-06a0f71de2df', 2, 'momma with puppies'),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', '98179ab7-8cd9-4d05-a6c8-24df846b8dd2', 3, 'brown puppy'),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', '41471973-9595-470f-b299-30a3423e267e', 4, ''),
+  ('4ed29ffe-150f-42ce-b0a9-2aadde646bcb', '53d4c4bd-9802-41ad-82ae-1e82734d56fc', 5, '');
 -- name: delete-collection_items
 DELETE FROM collection_items;
 
diff --git a/urls_test.go b/urls_test.go
index c6d53e5..29e9078 100644
--- a/urls_test.go
+++ b/urls_test.go
@@ -19,7 +19,7 @@ func TestListUrls(t *testing.T) {
 		t.Errorf(err.Error())
 	}
 
-	if len(urls) != 3 {
+	if len(urls) != 9 {
 		t.Errorf("urls length mismatch")
 	}