using new endpoints

pyOfferUp/__init__.py

@@ -1,2 +1,2 @@
-from pyOfferUp.fetch import get_posts, get_posts_by_lat_lon, driver_executable_path
+from pyOfferUp.fetch import get_listings, get_listings_by_lat_lon, get_listing_details
 import pyOfferUp.places

pyOfferUp/constants.py

@@ -0,0 +1,30 @@
+import enum
+
+ENDPOINT = "https://offerup.com"
+
+
+class CONDITION(enum.Enum):
+    NEW = "NEW"
+    OPEN_BOX = "OPEN_BOX"
+    REFURBISHED = "REFURBISHED"
+    USED = "USED"
+    BROKEN = "BROKEN"
+    OTHER = "OTHER"
+
+
+class SORT(enum.Enum):
+    NEWEST_FIRST = "-posted"
+    CLOSEST_FIRST = "distance"
+    PRICE_LOW_TO_HIGH = "price"
+    PRICE_HIGH_TO_LOW = "-price"
+
+
+class DELIVERY(enum.Enum):
+    PICKUP = "p"
+    SHIPPING = "s"
+    PICKUP_AND_SHIPPING = "p_s"
+
+
+class GRAPHQL(enum.Enum):
+    MODULAR_FEED = "query GetModularFeed($searchParams: [SearchParam]) {\n    modularFeed(params: $searchParams) {\n        looseTiles {\n            ...modularTileBanner\n            ...modularTileJob\n            ...modularTileEmptyState\n            ...modularTileListing\n            ...modularTileLocalDisplayAd\n            ...modularTileSearchAlert\n            ...modularTileSellerAd\n        }\n    }\n}\n\nfragment modularTileBanner on ModularFeedTileBanner {\n    tileId\n    tileType\n    title\n}\n\nfragment modularTileJob on ModularFeedTileJob {\n    tileId\n    tileType\n    job {\n        address {\n            city\n            state\n            zipcode\n        }\n        companyName\n        datePosted\n        image {\n            height\n            url\n            width\n        }\n        industry\n        jobId\n        jobListingUrl\n        pills {\n            text\n            type\n        }\n        title\n    }\n}\n\nfragment modularTileEmptyState on ModularFeedTileEmptyState {\n    tileId\n    tileType\n    title\n    description\n    iconType\n}\n\nfragment modularTileListing on ModularFeedTileListing {\n    tileId\n    listing {\n        ...modularListing\n    }\n    tileType\n}\n\nfragment modularListing on ModularFeedListing {\n    listingId\n    conditionText\n    flags\n    image {\n        height\n        url\n        width\n    }\n    isFirmPrice\n    locationName\n    price\n    title\n    vehicleMiles\n}\n\nfragment modularTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n    tileId\n    localDisplayAd {\n        ouAdId\n        adExperimentId\n        adNetwork\n        adRequestId\n        adTileType\n        advertiserId\n        businessName\n        callToAction\n        callToActionType\n        clickFeedbackUrl\n        contentUrl\n        experimentDataHash\n        headline\n        image {\n            height\n            url\n            width\n        }\n        impressionFeedbackUrl\n        searchId\n    }\n    tileType\n}\n\nfragment modularTileSearchAlert on ModularFeedTileSearchAlert {\n    tileId\n    tileType\n    title\n}\n\nfragment modularTileSellerAd on ModularFeedTileSellerAd {\n    tileId\n    listing {\n        ...modularListing\n    }\n    sellerAd {\n        ouAdId\n        adId\n        adExperimentId\n        adNetwork\n        adRequestId\n        adTileType\n        clickFeedbackUrl\n        experimentDataHash\n        impressionFeedbackUrl\n        searchId\n    }\n    tileType\n}"
+    LISTING_DETAIL = "query GetListingDetailByListingId($listingId: ID!, $isLoggedIn: Boolean = false, $deviceLocation: DeviceLocation) {\n  listing(listingId: $listingId, deviceLocation: $deviceLocation) {\n    ...listingDetail\n    __typename\n  }\n}\n\nfragment listingDetail on Listing {\n  id\n  badges\n  condition\n  description\n  discussionCount\n  distance {\n    unit\n    value\n    __typename\n  }\n  extractedAttributes {\n    attributeName\n    attributeValue\n    attributeValueSource\n    __typename\n  }\n  fulfillmentDetails {\n    buyItNowEnabled\n    canShipToBuyer\n    estimatedDeliveryDateEnd\n    estimatedDeliveryDateStart\n    localPickupEnabled\n    sellerPaysShipping\n    shippingEnabled\n    shippingParcelId\n    shippingPrice\n    showAsShipped\n    __typename\n  }\n  isFirmOnPrice\n  isLocal\n  isMerchantItem\n  lastEdited\n  listingCategory {\n    categoryAttributeMap {\n      attributeName\n      attributePriority\n      attributeUILabel\n      attributeValue\n      attributeValueSource\n      __typename\n    }\n    categoryV2 {\n      id\n      l1Id\n      l1Name\n      l2Id\n      l2Name\n      l3Id\n      l3Name\n      name\n      __typename\n    }\n    __typename\n  }\n  listingDetailAds {\n    adExperimentId\n    adTiles {\n      ...adsGoogleDisplayAd\n      __typename\n    }\n    placements {\n      adTileConfigs {\n        adSize {\n          additionalSizes\n          height\n          width\n          __typename\n        }\n        displayType\n        renderLocation\n        __typename\n      }\n      adType\n      quantity\n      __typename\n    }\n    __typename\n  }\n  listingId\n  locationDetails {\n    distance\n    latitude\n    locationName\n    longitude\n    __typename\n  }\n  merchantId\n  merchantProfile {\n    avatar {\n      small {\n        url\n        __typename\n      }\n      __typename\n    }\n    description\n    emailAddress\n    legacyUserOwner\n    phoneNumber\n    publicLocationName\n    storeName\n    ratingSummary {\n      average\n      count\n      __typename\n    }\n    __typename\n  }\n  originalPrice\n  originalTitle\n  owner {\n    id\n    profile {\n      avatars {\n        squareImage\n        __typename\n      }\n      c2cPhoneNumber {\n        countryCode\n        nationalNumber\n        __typename\n      }\n      clickToCallEnabled\n      dateJoined\n      isAutosDealer\n      isSubPrimeDealer\n      isTruyouVerified\n      name\n      notActive\n      openingHours {\n        day\n        hours\n        __typename\n      }\n      phoneNumber\n      publicLocation {\n        formattedAddress\n        latitude\n        longitude\n        name\n        __typename\n      }\n      publicLocationName\n      itemsPurchased\n      itemsSold\n      responseTime\n      ratingSummary {\n        average\n        count\n        __typename\n      }\n      reviews {\n        attributionIcon\n        average\n        readMoreUrl\n        title\n        userReviews {\n          profilePhotoUrl\n          text\n          __typename\n        }\n        __typename\n      }\n      websiteLink\n      __typename\n    }\n    __typename\n  }\n  ownerId\n  photos {\n    uuid\n    detail {\n      height\n      url\n      width\n      __typename\n    }\n    detailFull {\n      url\n      width\n      height\n      __typename\n    }\n    detailSquare {\n      height\n      url\n      width\n      __typename\n    }\n    list {\n      height\n      url\n      width\n      __typename\n    }\n    medium {\n      height\n      url\n      width\n      __typename\n    }\n    squareMedium {\n      height\n      url\n      width\n      __typename\n    }\n    __typename\n  }\n  postDate\n  price\n  quantity\n  saved @include(if: $isLoggedIn)\n  shippingOptions {\n    maxHandlingDays\n    maxShippingDays\n    minHandlingDays\n    minShippingDays\n    name\n    price\n    priority\n    __typename\n  }\n  shippingRate {\n    maxDeliveryDays\n    maxEstimatedDeliveryDate\n    minDeliveryDays\n    minEstimatedDeliveryDate\n    price\n    priority\n    __typename\n  }\n  sku\n  state\n  title\n  vehicleAttributes {\n    vehicleBody\n    vehicleCityMpg\n    vehicleColor\n    vehicleDriveTrain\n    vehicleDriveTrainClean\n    vehicleEngineCylinders\n    vehicleEpaCity\n    vehicleEpaHighway\n    vehicleExternalHistoryReport {\n      epochDate\n      imageUrl\n      issues\n      price {\n        microUnits\n        __typename\n      }\n      providerName\n      reportUrl\n      source\n      __typename\n    }\n    vehicleFuelType\n    vehicleFundamentals\n    vehicleHighwayMpg\n    vehicleId\n    vehicleMake\n    vehicleMiles\n    vehicleModel\n    vehicleStyleDisplay\n    vehicleTitleStatus\n    vehicleTransmission\n    vehicleTransmissionClean\n    vehicleTrim\n    vehicleVin\n    vehicleYear\n    __typename\n  }\n  __typename\n}\n\nfragment adsGoogleDisplayAd on GoogleDisplayAd {\n  ...baseGoogleDisplayAd\n  renderLocation\n  tileType\n  __typename\n}\n\nfragment baseGoogleDisplayAd on GoogleDisplayAd {\n  ouAdId\n  adExperimentId\n  adHeight\n  adMediationId\n  adNetwork\n  adRequestId\n  adWidth\n  clickFeedbackUrl\n  clientId\n  contentUrl\n  customTargeting {\n    key\n    values\n    __typename\n  }\n  displayAdType\n  formatIds\n  errorDrawable {\n    actionPath\n    listImage {\n      height\n      url\n      width\n      __typename\n    }\n    __typename\n  }\n  experimentDataHash\n  impressionFeedbackUrl\n  personalizationProperties {\n    key\n    values\n    __typename\n  }\n  type\n  __typename\n}\n"

pyOfferUp/fetch.py

@@ -1,66 +1,87 @@
-import enum
 import json
-import urllib
+import requests
 
-from selenium import webdriver
 from pyOfferUp import places
+from pyOfferUp.constants import *
 
-__driver__ = None
-driver_executable_path = None
 
+def __post_request__(request_body):
+    s = requests.Session()
+    response = s.get(ENDPOINT)
+    cookies = dict(response.cookies)
+    response = s.post(ENDPOINT + "/api/graphql",
+                      request_body,
+                      headers={"Content-Type": "application/json",
+                               "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"},
+                      verify=False,
+                      cookies=cookies)
 
-class SORT_BY(enum.Enum):
-    NEWEST_FIRST = "-posted"
-    CLOSEST_FIRST = "distance"
-    PRICE_LOW_TO_HIGH = "price"
-    PRICE_HIGH_TO_LOW = "-price"
+    data: dict = {}
 
+    if response.status_code == 200:
+        data = json.loads(response.content)
 
-class PURCHASE_BY(enum.Enum):
-    PICKUP = "p"
-    SHIPPING = "s"
-    PICKUP_AND_SHIPPING = "p_s"
+    return data
 
 
-def __get_driver__():
-    global __driver__
+# get state and city from places.py
+def get_listings(query, state, city=None, limit=50, pickup_distance=50, price_min=None, price_max=None,
+                 sort: SORT = SORT.NEWEST_FIRST,
+                 delivery: DELIVERY = DELIVERY.PICKUP,
+                 conditions=None):
+    if conditions is None:
+        conditions = []
 
-    if __driver__ is not None:
-        return __driver__
+    lat, lon = places.get_lat_lon(state, city)
+    return get_listings_by_lat_lon(query, lat, lon, limit, pickup_distance, price_min, price_max, sort, delivery,
+                                   conditions)
 
-    if driver_executable_path is None:
-        raise Exception("set fetch.driver_executable_path")
 
-    options = webdriver.ChromeOptions()
-    user_agent = 'MMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'
-    options.add_argument(f'user-agent={user_agent}')
-    options.add_argument('--headless')
+# get posts by latitude and longitude
+def get_listings_by_lat_lon(query, lat, lon, limit=50, pickup_distance=50, price_min=None, price_max=None,
+                            sort: SORT = SORT.NEWEST_FIRST,
+                            delivery: DELIVERY = DELIVERY.PICKUP,
+                            conditions: list[CONDITION] = None):
+    if conditions is None:
+        conditions = []
 
-    __driver__ = webdriver.Chrome(executable_path=driver_executable_path,
-                                  chrome_options=options)
+    search_params = [{"key": "q", "value": query},
+                     {"key": "distance", "value": str(pickup_distance)},
+                     {"key": "platform", "value": "web"},
+                     {"key": "lon", "value": str(lon)},
+                     {"key": "lat", "value": str(lat)},
+                     {"key": "limit", "value": str(limit)},
+                     {"key": "SORT", "value": sort.value},
+                     {"key": "DELIVERY_FLAGS", "value": delivery.value},
+                     {"key": "searchSessionId", "value": ""}]
 
-    return __driver__
+    if price_min is not None:
+        search_params.append({"key": "PRICE_MIN", "value": price_min})
 
+    if price_max is not None:
+        search_params.append({"key": "PRICE_MAX", "value": price_max})
 
-# get state and city from places.py
-def get_posts(query, state, city=None, limit=50, pickup_distance=50, price_min=0, price_max=1000000,
-              sort: SORT_BY = SORT_BY.NEWEST_FIRST,
-              purchase_by: PURCHASE_BY = PURCHASE_BY.PICKUP):
-    lat, lon = places.get_lat_lon(state, city)
-    return get_posts_by_lat_lon(query, lat, lon, limit, pickup_distance, price_min, price_max, sort, purchase_by)
+    if len(conditions) > 0:
+        conditions_str = ",".join([str(c.value) for c in conditions])
+        search_params.append({"key": "CONDITION", "value": conditions_str})
 
+    request_body = json.dumps({"operationName": "GetModularFeed", "variables": {"searchParams": search_params},
+                               "query": GRAPHQL.MODULAR_FEED.value})
 
-# get posts by latitude and longitude
-def get_posts_by_lat_lon(query, lat, lon, limit=50, pickup_distance=50, price_min=0, price_max=1000000,
-                         sort: SORT_BY = SORT_BY.NEWEST_FIRST,
-                         purchase_by: PURCHASE_BY = PURCHASE_BY.PICKUP):
-    api_url = "https://offerup.com/webapi/search/v4/feed/?lat={lat}&lon={lon}&limit={limit}" \
-              "&platform=web&experiment_id=experimentmodel24&q={query}&sort={sort}&radius={radius}" \
-              "&price_min={price_min}&price_max={price_max}&delivery_param={dp}" \
-        .format(lat=lat, lon=lon, limit=limit, query=urllib.parse.quote(query), sort=sort.value, radius=pickup_distance,
-                price_min=price_min, price_max=price_max, dp=purchase_by)
-
-    driver = __get_driver__()
-    driver.get(api_url)
-    pre = driver.find_element_by_tag_name("pre").text
-    return json.loads(pre)['data']['feed_items']
+    response = __post_request__(request_body)
+    listings = []
+
+    if 'data' in response and 'modularFeed' in response['data'] and 'looseTiles' in response['data']['modularFeed']:
+        listings = [lt['listing'] for lt in response['data']['modularFeed']['looseTiles'] if 'listing' in lt]
+
+        for listing in listings:
+            listing['listingUrl'] = ENDPOINT + "/item/detail/" + listing['listingId']
+
+    return listings
+
+
+def get_listing_details(listing_id):
+    variables = {"isLoggedIn": False, "listingId": listing_id}
+    request_body = json.dumps(
+        {"operationName": "GetListingDetailByListingId", "variables": variables, "query": GRAPHQL.LISTING_DETAIL.value})
+    return __post_request__(request_body)

setup.py

@@ -11,8 +11,8 @@
     url='https://github.com/oscar0812/pyOfferUp',  # Provide either the link to your github or to your website
     download_url='https://github.com/oscar0812/pyOfferUp/archive/refs/tags/v_01.tar.gz',  # I explain this later on
     keywords=['OFFERUP', 'BUYING', 'SCRAPER', 'BITTLE'],  # Keywords that define your package best
-    install_requires=[  # I get to this in a second
-        'selenium'
+    install_requires=[
+        'requests'
     ],
     classifiers=[
         'Development Status :: 3 - Alpha',