API tools faq
paste
Advertisement
Sushill

OfferUp product url listings

Sushill
Jun 18th, 2025 (edited)
540
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 13.40 KB | Source Code | 0 0
raw download clone embed print report
  1. # add headers and cookies
  2.  
  3. headers = {}
  4. cookies = {}
  5.  
  6. query = """
  7.    query GetModularFeed($searchParams: [SearchParam], $debug: Boolean = false) {\n  modularFeed(params: $searchParams, debug: $debug) {\n    analyticsData {\n      requestId\n      searchPerformedEventUniqueId\n      searchSessionId\n      __typename\n    }\n    categoryInfo {\n      categoryId\n      isForcedCategory\n      __typename\n    }\n    feedAdditions\n    filters {\n      ...modularFilterNumericRange\n      ...modularFilterSelectionList\n      __typename\n    }\n    legacyFeedOptions {\n      ...legacyFeedOptionListSelection\n      ...legacyFeedOptionNumericRange\n      __typename\n    }\n    looseTiles {\n      ...modularTileBanner\n      ...modularTileBingAd\n      ...modularTileGoogleDisplayAd\n      ...modularTileJob\n      ...modularTileEmptyState\n      ...modularTileListing\n      ...modularTileLocalDisplayAd\n      ...modularTileSearchAlert\n      ...modularTileSellerAd\n      ...modularModuleTileAdsPostXAd\n      __typename\n    }\n    modules {\n      ...modularGridModule\n      __typename\n    }\n    pageCursor\n    query {\n      ...modularQueryInfo\n      __typename\n    }\n    requestTimeMetadata {\n      resolverComputationTimeSeconds\n      serviceRequestTimeSeconds\n      totalResolverTimeSeconds\n      __typename\n    }\n    searchAlert {\n      alertId\n      alertStatus\n      searchAlertCount\n      __typename\n    }\n    personalizationPath\n    debugInformation @include(if: $debug) {\n      rankedListings {\n        listingId\n        attributes {\n          key\n          value\n          __typename\n        }\n        __typename\n      }\n      lastViewedItems {\n        listingId\n        attributes {\n          key\n          value\n          __typename\n        }\n        __typename\n      }\n      categoryAffinities {\n        affinity\n        count\n        decay\n        affinityOwner\n        __typename\n      }\n      rankingStats {\n        key\n        value\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment modularFilterNumericRange on ModularFeedNumericRangeFilter {\n  isExpandedHighlight\n  lowerBound {\n    ...modularFilterNumericRangeBound\n    __typename\n  }\n  shortcutLabel\n  shortcutRank\n  subTitle\n  targetName\n  title\n  type\n  upperBound {\n    ...modularFilterNumericRangeBound\n    __typename\n  }\n  __typename\n}\n\nfragment modularFilterNumericRangeBound on ModularFeedNumericRangeFilterNumericRangeBound {\n  label\n  limit\n  placeholderText\n  targetName\n  value\n  __typename\n}\n\nfragment modularFilterSelectionList on ModularFeedSelectionListFilter {\n  targetName\n  title\n  subTitle\n  shortcutLabel\n  shortcutRank\n  type\n  isExpandedHighlight\n  options {\n    ...modularFilterSelectionListOption\n    __typename\n  }\n  __typename\n}\n\nfragment modularFilterSelectionListOption on ModularFeedSelectionListFilterOption {\n  isDefault\n  isSelected\n  label\n  subLabel\n  value\n  __typename\n}\n\nfragment legacyFeedOptionListSelection on FeedOptionListSelection {\n  label\n  labelShort\n  name\n  options {\n    default\n    label\n    labelShort\n    selected\n    subLabel\n    value\n    __typename\n  }\n  position\n  queryParam\n  type\n  __typename\n}\n\nfragment legacyFeedOptionNumericRange on FeedOptionNumericRange {\n  label\n  labelShort\n  leftQueryParam\n  lowerBound\n  name\n  options {\n    currentValue\n    label\n    textHint\n    __typename\n  }\n  position\n  rightQueryParam\n  type\n  units\n  upperBound\n  __typename\n}\n\nfragment modularTileBanner on ModularFeedTileBanner {\n  tileId\n  tileType\n  title\n  __typename\n}\n\nfragment modularTileBingAd on ModularFeedTileBingAd {\n  tileId\n  bingAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    adSettings {\n      repeatClickRefractoryPeriodMillis\n      collapsible\n      __typename\n    }\n    bingClientId\n    clickFeedbackUrl\n    clickReturnUrl\n    contentUrl\n    deepLinkEnabled\n    experimentDataHash\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    impressionFeedbackUrl\n    impressionUrls\n    viewableImpressionUrls\n    installmentInfo {\n      amount\n      description\n      downPayment\n      __typename\n    }\n    itemName\n    lowPrice\n    price\n    searchId\n    sellerName\n    templateFields {\n      key\n      value\n      __typename\n    }\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n  tileId\n  googleDisplayAd {\n    ouAdId\n    additionalSizes\n    adExperimentId\n    adHeight\n    adNetwork\n    adPage\n    adRequestId\n    adTileType\n    adWidth\n    adaptive\n    channel\n    clickFeedbackUrl\n    clientId\n    contentUrl\n    customTargeting {\n      key\n      values\n      __typename\n    }\n    displayAdType\n    errorDrawable {\n      actionPath\n      listImage {\n        height\n        url\n        width\n        __typename\n      }\n      __typename\n    }\n    experimentDataHash\n    formatIds\n    impressionFeedbackUrl\n    personalizationProperties {\n      key\n      values\n      __typename\n    }\n    prebidConfigs {\n      key\n      values {\n        timeout\n        tamSlotUUID\n        liftoffPlacementIDs\n        nimbusPriceMapping\n        adPosition\n        __typename\n      }\n      __typename\n    }\n    renderLocation\n    searchId\n    searchQuery\n    templateId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileJob on ModularFeedTileJob {\n  tileId\n  tileType\n  job {\n    address {\n      city\n      state\n      zipcode\n      __typename\n    }\n    companyName\n    datePosted\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    industry\n    jobId\n    jobListingUrl\n    jobOwnerId\n    pills {\n      text\n      type\n      __typename\n    }\n    title\n    apply {\n      method\n      value\n      __typename\n    }\n    wageDisplayValue\n    provider\n    __typename\n  }\n  __typename\n}\n\nfragment modularTileEmptyState on ModularFeedTileEmptyState {\n  tileId\n  tileType\n  title\n  description\n  iconType\n  __typename\n}\n\nfragment modularTileListing on ModularFeedTileListing {\n  tileId\n  listing {\n    ...modularListing\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularListing on ModularFeedListing {\n  listingId\n  conditionText\n  flags\n  image {\n    height\n    url\n    width\n    __typename\n  }\n  isFirmPrice\n  locationName\n  price\n  title\n  vehicleMiles\n  __typename\n}\n\nfragment modularTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n  tileId\n  localDisplayAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    advertiserId\n    businessName\n    callToAction\n    callToActionType\n    clickFeedbackUrl\n    contentUrl\n    experimentDataHash\n    headline\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    impressionFeedbackUrl\n    searchId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileSearchAlert on ModularFeedTileSearchAlert {\n  tileId\n  tileType\n  title\n  __typename\n}\n\nfragment modularTileSellerAd on ModularFeedTileSellerAd {\n  tileId\n  listing {\n    ...modularListing\n    __typename\n  }\n  sellerAd {\n    ouAdId\n    adId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    clickFeedbackUrl\n    experimentDataHash\n    impressionFeedbackUrl\n    searchId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularModuleTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n  ...modularTileAdsPostXAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n  tileId\n  adsPostXAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    clickFeedbackUrl\n    experimentDataHash\n    impressionFeedbackUrl\n    searchId\n    offer {\n      beacons {\n        noThanksClick\n        close\n        __typename\n      }\n      title\n      description\n      clickUrl\n      image\n      pixel\n      ctaYes\n      ctaNo\n      perkswallLabel\n      perkswallUrl\n      __typename\n    }\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularGridModule on ModularFeedModuleGrid {\n  moduleId\n  collection\n  formFactor\n  grid {\n    actionPath\n    tiles {\n      ...modularModuleTileBingAd\n      ...modularModuleTileGoogleDisplayAd\n      ...modularModuleTileListing\n      ...modularModuleTileLocalDisplayAd\n      ...modularModuleTileSellerAd\n      __typename\n    }\n    __typename\n  }\n  moduleType\n  rank\n  rowIndex\n  searchId\n  subTitle\n  title\n  infoActionPath\n  feedIndex\n  __typename\n}\n\nfragment modularModuleTileBingAd on ModularFeedTileBingAd {\n  ...modularTileBingAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n  ...modularTileGoogleDisplayAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileListing on ModularFeedTileListing {\n  ...modularTileListing\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n  ...modularTileLocalDisplayAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileSellerAd on ModularFeedTileSellerAd {\n  ...modularTileSellerAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularQueryInfo on ModularFeedQueryInfo {\n  appliedQuery\n  decisionType\n  originalQuery\n  suggestedQuery\n  __typename\n}\n
  8.    """
  9.  
  10.     random_delay = await random_interval()
  11.     all_listings = []
  12.     page_cursor = None
  13.     current_page = 0
  14.     search_session_id = '579cfbd4-193d-44d8-8ba5-b5cefeabe928' # Get search_session_id from the website's headers or may be it's in payload
  15.  
  16.     # Loop to fetch multiple pages
  17.     while current_page < max_pages:
  18.         # print(f"Scrolled {count+1} pages out of {max_pages}")
  19.         # Prepare search parameters - always include the latest page_cursor
  20.         search_params = [
  21.             {'key': 'platform', 'value': 'web'},
  22.             {'key': 'lon', 'value': '-77.1995'},
  23.             {'key': 'lat', 'value': '38.696'},
  24.             {'key': 'zipcode', 'value': '33503'},
  25.             {'key': 'DISTANCE', 'value': '200'}, # 200 in miles
  26.             {'key': 'cid', 'value': str(category_id)}, # category id is based on the category of the listings, please refer to the network tools of the browser
  27.             {'key': 'limit', 'value': '50'},
  28.             {'key': 'searchSessionId', 'value': search_session_id}
  29.         ]
  30.  
  31.         # Page Cursor is generate after each scroll
  32.         if page_cursor:
  33.             search_params.append({'key': 'page_cursor', 'value': page_cursor})
  34.  
  35.         json_data = {
  36.             'operationName': 'GetModularFeed',
  37.             'variables': {
  38.                 'debug': False,
  39.                 'searchParams': search_params,
  40.             },
  41.             'query': query,
  42.         }
  43.        
  44.         try:
  45.             # I have create a make_requests function sepearately, please use your own preferred requests module. I have used curl_cffi one.
  46.             response = await make_requests(
  47.                 'post',
  48.                 'https://offerup.com/api/graphql',
  49.                 headers=headers,
  50.                 cookies=cookies,
  51.                 json=json_data,
  52.  
  53.             )
  54.            
  55.             data = response.json()
  56.  
  57.             # Check if we have valid data
  58.             if 'data' not in data or 'modularFeed' not in data['data'] or 'looseTiles' not in data['data']['modularFeed']:
  59.                 print(f"Invalid or empty response on page {current_page + 1}")
  60.                 break
  61.  
  62.             # Get the listings from this page
  63.             page_listings = []
  64.             for tile in data['data']['modularFeed']['looseTiles']:
  65.                 # Check if this is a listing tile (not an ad or other content)
  66.                 if 'listing' in tile:
  67.                     print(tile['listing']['title'])
  68.                     listing_info = {
  69.                         'title': tile['listing']['title'],
  70.                         # 'id': tile['listing']['listingId'],
  71.                         'url': f"https://offerup.com/item/detail/{tile['listing']['listingId']}",
  72.                         'price': tile['listing']['price'],
  73.                         # 'location': tile['listing']['locationName']
  74.                     }
  75.                     page_listings.append(listing_info)
  76.  
  77.             # Add these listings to our main list
  78.             all_listings.extend(page_listings)
  79.             await asyncio.sleep(random_delay)
  80.  
  81.             print('--------------------------------------------------')
  82.             print(f"Page {current_page + 1}: Found {len(page_listings)} listings")
  83.             print('--------------------------------------------------\n')
  84.  
  85.             # Get the cursor for the next page
  86.             new_cursor = data['data']['modularFeed'].get('pageCursor')
  87.  
  88.             # If we have no new cursor or it's the same as the previous one, we're done
  89.             if not new_cursor or new_cursor == page_cursor:
  90.                 print("No more pages available")
  91.                 break
  92.  
  93.             # Update the cursor for the next request
  94.             page_cursor = new_cursor
  95.             current_page += 1
  96.  
  97.         except Exception as e:
  98.             print(f"Error fetching data: {str(e)}")
  99.             break
  100.  
  101.     return all_listings
Tags: offerUp
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!