open_api.yaml

---
  swagger: "2.0"
  info: 
    version: "0.0.1"
    title: "Data Together API"
    description: "Api for Data Together records"
    termsOfService: "https://archivers.co/terms/api"
    contact: 
      name: "b5"
    license: 
      name: "AGPL"
  host: "api.archivers.co"
  basePath: "/"
  schemes: 
  - "https"
  consumes: 
  - "application/json"
  produces: 
  - "application/json"
  paths:
    /users:
      get:
        description: "Get Users"
        produces:
          - "application/json"
        responses:
          "200":
            description: "Enveloped array of Users"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  type: array
                  items:
                    $ref: "#/definitions/User"
                pagination:
                  type: object
    /users/{id}:
      get:
        description: "Get info on a single user"
        produces:
        - application/json
        consumes:
        - application/json
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: "Single User"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/User"
    /primers:
      get:
        description: "List Primers"
        produces:
        - "application/json"
        responses:
          "200":
            description: "Enveloped array of Primers"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  type: array
                  items:
                    $ref: "#/definitions/Primer"
                pagination:
                  type: object
    /primers/{id}:
      get:
        description: "Get a Primer"
        produces:
        - "application/json"
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: "List"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/Primer"
    /sources:
      get:
        description: "List Sources"
        produces:
        - "application/json"
        responses:
          "200":
            description: "Enveloped Sources"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  type: array
                  items:
                    $ref: "#/definitions/Source"
                pagination:
                  type: object
    /sources/{id}:
      get:
        description: "Get a Source"
        produces:
        - "application/json"
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: "Enveloped Source"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/Source"
    /urls:
      get:
        description: "List Urls"
        produces:
        - "application/json"
        responses:
          "200":
            description: "Enveloped List of Urls"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  type: array
                  items:
                    $ref: "#/definitions/Url"
                pagination:
                  type: object
    /urls/{id}:
      get:
        description: "Get a Url"
        produces:
        - "application/json"
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: "Enveloped Url"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/Url"
    # /links:
    #   get:
    #     description: "get links"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #             pagination:
    #               type: object
    # /links/:
    #   get:
    #     description: "get links"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #             pagination:
    #               type: object
    # /content:
    #   get:
    #     description: "get content"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #             pagination:
    #               type: object
    # /content/:
    #   get:
    #     description: "get content/Z"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #             pagination:
    #               type: object
    # /content/{subjectHash}:
    #   get:
    #     description: "get metadata"
    #     produces:
    #       - "application/json"
    #       parameters:
    #       - name: id
    #         in: path
    #         required: true
    #         type: string
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: array
    #             pagination:
    #               type: object
    # /metadata/{subjectHash}/{hash}:
    #   get:
    #     description: "Get metadata for a given subject hash"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #               $ref: "#/definitions/Metadata"
    # /consensus:
    #   get:
    #     description: "get consensus"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: array
    #               $ref: "#/definitions/Consensus"
    #             pagination:
    #               type: object
    # /consensus/{subjectHash}:
    #   get:
    #     description: "Get Consensus for a subject hash"
    #     produces:
    #       - "application/json"
    #     responses:
    #       "200":
    #         description: "List"
    #         schema:
    #           type: object
    #           properties:
    #             meta:
    #               type: object
    #             data:
    #               type: object
    #               $ref: "#/definitions/Consensus"
    /repositories:
      get:
        description: List Data Repositories
        produces:
        - application/json
        responses:
          "200":
            description: list of repositories
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  type: array
                  items:
                    $ref: "#/definitions/DataRepository"
                pagination:
                  type: object
    /repository/{id}:
      get:
        description: get details for a single data repository
        produces:
        - application/json
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: repository details
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/DataRepository"
    /coverage:
      get:
        description: Coverage tree of url-based resources
        produces:
        - application/json
        parameters:
        - name: primer
          in: query
          type: string
          description: id of the primer to get coverage for
        - name: depth
          in: query
          type: integer
          description: maximum number of children to return
        - name: repos
          in: query
          type: string
          description: comma-separated list of repository ids to limit coverage results for
          default: all
        - name: root
          in: query
          type: string
          description: url of node to use as the root of the coverage tree, useful for tree traversal in combination with the depth param
          default: ""
        responses:
          "200":
            description: covergage tree
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/CoverageTree"
                pagination:
                  type: object
    /collections:
      get:
        description: List Collections
        produces:
        - "application/json"
        responses:
          "200":
            description: "Enveloped array of collections"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/Collection"
                pagination:
                  type: object
    /collections/{id}:
      get:
        description: "Get a Collection"
        produces:
        - "application/json"
        parameters:
        - name: id
          in: path
          required: true
          type: string
        responses:
          "200":
            description: "Enveloped Collection Object"
            schema:
              type: object
              properties:
                meta:
                  type: object
                data:
                  $ref: "#/definitions/Collection"
  definitions:
    Collection:
      type: "object"
      required:
      - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: Created timestamp rounded to seconds in UTC
        updated:
          type: string
          format: date-time
          description: Updated timestamp rounded to seconds in UTC
        creator:
          type: string
          description: sha256 multihash of the public key that created this collection
          example: "358a2a6b8e857836a9410c3ae5285eb5fec6fda7dcb7c78f75b4bada99bceea3"
        title:
          type: string
          description: human-readable title of the collection
          example: "EPA Volatile Organic Compound Measurements"
        schema:
          type: array
          description: csv column headers, first value must always be "hash"
          items: 
            type: string
        contents:
          type: array
          description: actual collection contents
          items:
            type: string
    # Consensus:
    #   type: "object"
    #   required:
    #     - "id"
    CoverageTree:
      type: object
      description: Tree of url components showing coverage information
      properties:
        numLeaves:
          type: integer
          description: number of nodes in the tree that have no children. full url endpoints are leaves.
          example: 5000
        numLeavesArchived:
          type: integer
          description: number of leaves that have been archived by at least one data repository
          example: 3101
        children:
          type: array
          description: array of nodes that are a child of this node
          items:
            type: object
          example:
            name: "ftp://ftp.epa.gov"
            numLeaves: 3
            numLeavesArchived: 1
            numChildren: 2
            children: 
            - name: "castnet"
              numLeaves: 2
              numLeavesArchived: 1
              numChildren: 1
              children: [ ... ]
        coverage:
          type: array
          description: array of services that have coverage information for this node
          items:
            type: object
          example:
          - repositoryId: "4c0122g5-38a8-40b3-be91-c324bf686a87"
            archived: true
            priority: 10
          - repositoryId: "4c0122g5-38a8-40b3-be91-c324bf686a87"
            archived: false
            priority: 0
    DataRepository:
      type: object
      description: representation of a place that keeps an archive of data, eg the internet archive
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: timestamp of first addition to the database rounded to seconds in UTC.
        updated:
          type: string
          format: date-time
          description: timestamp of most recent change rounded to seconds in UTC
        title:
          type: string
          description: title of the data repository
          example: The Internet Archive
        description:
          type: string
          description: description of the repository
          example: the archive of the worlds internet
        url:
          type: string
          description: url to home page of the repositoriy
          example: https://archive.org
    Link:
      type: object
      required:
      - created
      - updated
      - src
      - dst
      properties:
        created:
          format: string
          description: created timestamp rounded to seconds in UTC
        updated:
          format: string
          description: updated timestamp rounded to seconds in UTC
        src:
          format: string
          description: origin url of the linking document
        dst:
          format: string
          description: absolute url of the <a> href property
    Metadata:
      type: "object"
      required:
        - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        hash:
          type: string
          description: sha256 multihash of all other fields in metadata as expressed by Metadata.HashableBytes()
        timestamp:
          type: string
          format: date-time
          description: Creation timestamp
        keyId:
          type: string
          description: Sha256 multihash of the public key that signed this metadata
        subject:
          type: string
          description: Sha256 multihash of the content this metadata is describing
        prev:
          type: string
          description: Hash value of the metadata that came before this, if any
        meta:
          type: object
          description: Acutal stored metadata about the subject, can be any valid json Object
    Primer:
      type: "object"
      required:
      - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: timestamp of first addition to the database rounded to seconds in UTC.
        updated:
          type: string
          format: date-time
          description: timestamp of most recent change rounded to seconds in UTC
        shortTitle:
          type: string
          description: >
           shortest possible expression of this primer's name, usually an acronym
           called shortTitle b/c acronyms collide often & users should feel free to
           expand on acronyms 
          example: EPA
        title:
          type: string
          description: human-readable title of this primer. 
          example: "Environmental Protection Agency"
        description:
          type: string
          description: long-form description of this primer. 
          example: >
            The mission of the Environmental Protection Agency is to protect human health 
            and the environment through the development and enforcement of regulations. 
            The EPA is responsible for administering a number of laws that span various sectors, 
            such as agriculture, transportation, utilities, construction, and oil and gas. 
            In the budget for FY 2017, the agency lays out goals to better support communities 
            and address climate change following the President’s Climate Action Plan. 
            Additionally, the agency aims to improve community water infrastructure, chemical 
            plant safety, and collaborative partnerships among federal, state, and tribal levels.
        parent:
          type: object
          description: parent primer (if any)
          example:
            null
        subPrimers:
          type: array
          description: child-primers list
        meta:
          type: object
          description: metadata to associate with this primer
          example:
            county: US
            primaryLanguage: en
        stats:
          type: object
          description: statistics about this primer
          # TODO
          # example:
        sources:
          type: array
          description: collection of child sources
          items:
            $ref : "#/definitions/Source"
    Source:
      type: object
      description: >
        Source is a concreate handle for archiving. Crawlers use
        source's url as a base of a link tree. Sources are connected
        to a parent Primer to provide context & organization.
      required:
      - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: timestamp of first addition to the database rounded to seconds in UTC.
        updated:
          type: string
          format: date-time
          description: timestamp of most recent change rounded to seconds in UTC
        title:
          type: string
          description: human-readable title for this source
          example: "epa.gov"
        description:
          type: string
          description: description of the source, ideally one paragraph
          example: entire epa site
        url:
          type: string
          description: url to serve as boundaries for archiving
          example: "epa.gov"
        primerId:
          type: string
          description: primer this source is connected to
          example: "5b1031f4-38a8-40b3-be91-c324bf686a87"
        crawl:
          type: boolean
          description: weather or not this url should be crawled be a web crawler
          example: true
        staleDuration:
          type: integer
          description: amount of time before a link within this tree is considered in need of re-checking for changes. currently not in use, but planned.
        lastAlertSent:
          type: string
          description: yeah this'll probably get depricated. Part of a half-baked alerts feature idea.
        meta:
          type: object
          description: Metadata associated with this source that should be added to all child urls, currently not in use, but planned
        stats:
          type: object
          description: Stats about this source
    Url:
      type: "object"
      required:
      - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: timestamp of first addition to the database rounded to seconds in UTC.
        updated:
          type: string
          format: date-time
          description: timestamp of most recent change rounded to seconds in UTC
        url:
          type: string
          description: URI string without any normalization. Url strings must always be absolute. unique to each entry
          example: http://www.epa.gov
        lastGet:
          type: string
          format: date-time
          description: timestamp for most recent GET request
        lastHead:
          type: string
          format: date-time
          description: timestamp for most recent HEAD request
        status:
          type: integer
          description: latest returned HTTP response status code
        contentType:
          type: string
          description: latest returnd 'Content-Type' HTTP header
          example: text/html; charset=utf-8
        contentSniff:
          type: string
          description: Result of mime sniffing to GET response body, as detailed at https://mimesniff.spec.whatwg.org
          example: text/html; charset=utf-8
        contentLength:
          type: integer
          description: server-specified ContentLength header
          example: -1
        title:
          type: string
          description: HTML Title tag attribute
          example: United States Environmental Protection Agency, US EPA
        downloadTook: 
          type: integer
          description: Time remote server took to transfer content in miliseconds.
          example: 0
        headersTook:
          type: integer
          description: Time taken to  in miliseconds. currently not implemented
          example: 0
        headers:
          type: string
          description: key-value array of returned headers from most recent HEAD or GET request stored in the form [key,value,key,value...]
          example:
            ["X-Content-Type-Options","nosniff","Expires","Fri, 24 Feb 2017 21:53:45 GMT","Date","Fri, 24 Feb 2017 21:53:45 GMT","Etag","W/\"7f53-549471782bb42\"","X-Ua-Compatible","IE=Edge,chrome=1","X-Cached-By","Boost","Content-Type","text/html; charset=utf-8","Vary","Accept-Encoding","Accept-Ranges","bytes","Cache-Control","no-cache, no-store, must-revalidate, post-check=0, pre-check=0","Server","Apache","Connection","keep-alive","Strict-Transport-Security","max-age=31536000; preload;"]
        meta: 
          type: array
          description: any associative metadata for this url
        hash:
          type: string
          description: Hash is a multihash sha-256 of the response body of a GET request
          example: "1220459219b10032cc86dcdbc0f83aea15a9d3e1119e7b5170beaee233008ea2c2de"
        contentUrl:
          type: string
          description: Url to saved content
          example: https://content.archivers.co/1220459219b10032cc86dcdbc0f83aea15a9d3e1119e7b5170beaee233008ea2c2de
    User:
      type: "object"
      required:
      - "id"
      properties:
        id:
          $ref: "#/definitions/UUID"
        created:
          type: string
          format: date-time
          description: timestamp of first addition to the database rounded to seconds in UTC.
        updated:
          type: string
          format: date-time
          description: timestamp of most recent change rounded to seconds in UTC
        username:
          type: string
          description: handle for the user. min 1 character, max 80. composed of [_,-,a-z,A-Z,1-9]
          example: "wonder_woman"
        type:
          type: string
          description: specifies weather this is a user or an organization
          example: "user"
        name:
          type: string
          description: user name field. could be first[co]last, but not strictly enforced
          example: "Diana Prince"
        description:
          type: string
          description: user-filled description of self
          example: The Spirit of Truth
        homeUrl:
          type: string
          description: url this user wants the world to click
          example: "https://en.wikipedia.org/wiki/Wonder_Woman"
        currentKey:
          type: string
          description: sh256 multihash of public key that this user is currently using for signatures
          example: "358a2a6b8e857836a9410c3ae5285eb5fec6fda7dcb7c78f75b4bada99bceea3"
    UUID:
      type: string
      example: c98255ce-30a2-4fe5-94a6-7e6ec08a46ec
    Snapshot:
      type: "object"
      properties:
        url:
          type: string
          description: The url that was requested
        created:
          type: string
          format: date-time
          description: Time this request was issued
        status:
          type: integer
          description: Returned Status
        duration:
          type: integer
          description: Time to complete response in milliseconds
        headers:
          type: string
          description: Record of all returned headers in [key,value,key,value...]
        hash:
          type: string
          description: Multihash of response body (if any)
    Uncrawlable:
      type: "object"
      required:
      - "url"
      properties:
        id:
          $ref: "#/definitions/UUID"
        url:
          type: string 
          description: url from urls table, must be unique
        created:
          type: string 
          description: Created timestamp rounded to seconds in UTC
        updated:
          type: string 
          description: Updated timestamp rounded to seconds in UTC
        creator:
          type: string 
          description: sha256 multihash of the public key that created this uncrawlable
        name:
          type: string 
          description: name of person making submission
        email:
          type: string 
          description: email address of person making submission
        eventName:
          type: string 
          description: name of data rescue event where uncrawlable was added
        agency:
          type: string 
          description: agency name
        agencyId:
          type: string 
          description: EDGI agency Id
        subagencyId:
          type: string 
          description: EDGI subagency Id
        orgId:
          type: string 
          description: EDGI organization Id
        suborgId:
          type: string 
          description: EDGI Suborganization Id
        subprimerId:
          type: string 
          description: EDGI subprimer Id
        ftp:
          type: boolean 
          description: flag for ftp content
        database:
          type: boolean 
          description: flag for 'database'
        interactive:
          type: boolean 
          description: flag for visualization / interactive content obfuscating data
        manyFiles:
          type: boolean 
          description: flag for a page that links to many files
        comments:
          type: string 
          description: uncrawlable comments