API tools faq
paste
furas

Python - MPI - (FB: LearnPython)

furas
Dec 21st, 2016
193
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.06 KB | None | 0 0
raw download clone embed print report
  1. #!/usr/bin/env python2
  2.  
  3. #
  4. # https://pythonhosted.org/mpi4py/usrman/tutorial.html
  5. #
  6. # mpiexec -n 5 python script.py
  7. #
  8.  
  9. import requests
  10. from mpi4py import MPI
  11.  
  12. # --- functions ---
  13.  
  14. def build_ranges(value, numsplits, rank):
  15.    
  16.     print '[RANK: %d ] build_ranges: %s/%s' % (rank, value, numsplits)
  17.    
  18.     ranges = []
  19.  
  20.     for i in range(numsplits):
  21.         if i == 0:
  22.             start = i
  23.         else:
  24.             start = int(round(1 + i * value/(numsplits*1.0), 0))
  25.  
  26.         end = int(round(1 + i * value/(numsplits*1.0) + value/(numsplits*1.0)-1, 0))
  27.            
  28.         ranges.append((start, end))
  29.  
  30.     return ranges
  31.  
  32.  
  33. def get_file_data(url, data_range, rank):
  34.    
  35.     print '[RANK: %d ] get_file_data: %s' % (rank, data_range)
  36.    
  37.     response = requests.get(url, headers={'Range': 'bytes=%s-%s' % data_range})
  38.     data = response.content
  39.    
  40.     return data
  41.  
  42.  
  43. def merge(filename, data, data_range, rank):
  44.  
  45.     offset, end = data_range
  46.    
  47.     print '[RANK: %d ] merge: %s-%s' % (rank, offset, end)
  48.  
  49.     fh = MPI.File.Open(comm, filename, amode)
  50.     fh.Write_at_all(offset, data)
  51.     fh.Close()
  52.  
  53. # --- main ---
  54.  
  55. comm = MPI.COMM_WORLD
  56. rank = comm.Get_rank()
  57. numr = comm.Get_size()
  58. amode = MPI.MODE_WRONLY|MPI.MODE_CREATE
  59.  
  60. print '[RANK: %d ] starting: %d/%d' % (rank, rank, numr)
  61.  
  62. # ---------------------------------------------------------------------
  63.  
  64. #url = 'http://2u.fileshare.ro/download/3172391991/JerryCo+Feat.+Sanziana+Niculae+-+Esti+Tot+Ce+Am+%28+Original+Radio+Edit+%29+%5B+AlegeMuzica.Info+%5D.mp3'
  65. url = 'http://greenteapress.com/thinkpython/thinkpython.pdf'
  66. # url = 'http://www25.zippyshare.com/d/RTtv9Zv1/30183/Andra%20feat.%20David%20Bisbal%20-%20Without%20You%20%40%20PeMuzica.Com.mp3'
  67. #url = 'http://broadcast.lds.org/churchmusic/MP3/1/2/nowords/271.mp3'
  68.  
  69. filename = url.split('/')[-1]
  70.  
  71. # - different proceses -
  72.  
  73. if rank == 0:
  74.     if not url:
  75.         print "Please Enter some url to begin download."
  76.  
  77.     # - get file size -
  78.  
  79.     response = requests.head(url, headers={'Accept-Encoding': 'identity'})
  80.     size_in_bytes = response.headers.get('content-length', None)
  81.  
  82.     print '[RANK: %d ] %s bytes to download.' % (rank, size_in_bytes)
  83.  
  84.     if not size_in_bytes:
  85.         print "Size cannot be determined."
  86.         exit()
  87.        
  88.     # - generate ranges -
  89.    
  90.     byte_ranges = build_ranges(int(size_in_bytes), numr, rank)
  91.  
  92.     print '[RANK: %d ] byte_ranges: %s' % (rank, byte_ranges)
  93.  
  94.     # - send ranges -
  95.    
  96.     for itm in range(1, len(byte_ranges)):
  97.         req = comm.isend(byte_ranges[itm], dest=itm, tag=itm)
  98.         req.wait()
  99.  
  100.     # - "receive" range -
  101.  
  102.     data_range = byte_ranges[0]
  103.  
  104. else:
  105.     # - receive range -
  106.  
  107.     req = comm.irecv(source=0, tag=rank)
  108.     data_range = req.wait()
  109.  
  110. # - all processes -
  111.  
  112. print '[RANK: %d ] data_range: %s' % (rank, data_range)
  113.  
  114. # - download -
  115.  
  116. data = get_file_data(url, data_range, rank)
  117.  
  118. # - merge -
  119.  
  120. merge(filename, data, data_range, rank)
  121.  
  122. # - end -
  123.  
  124. print '[RANK: %d ] ending: %d/%d' % (rank, rank, numr)
  125.  
  126. # --- RESULT ---
  127.  
  128. '''
  129. [RANK: 1 ] starting: 1/5
  130. [RANK: 3 ] starting: 3/5
  131. [RANK: 4 ] starting: 4/5
  132. [RANK: 0 ] starting: 0/5
  133. [RANK: 2 ] starting: 2/5
  134. [RANK: 0 ] 834685 bytes to download.
  135. [RANK: 0 ] build_ranges: 834685/5
  136. [RANK: 0 ] byte_ranges: [(0, 166937), (166938, 333874), (333875, 500811), (500812, 667748), (667749, 834685)]
  137. [RANK: 0 ] data_range: (0, 166937)
  138. [RANK: 2 ] data_range: (333875, 500811)
  139. [RANK: 1 ] data_range: (166938, 333874)
  140. [RANK: 1 ] get_file_data: (166938, 333874)
  141. [RANK: 2 ] get_file_data: (333875, 500811)
  142. [RANK: 3 ] data_range: (500812, 667748)
  143. [RANK: 3 ] get_file_data: (500812, 667748)
  144. [RANK: 0 ] get_file_data: (0, 166937)
  145. [RANK: 4 ] data_range: (667749, 834685)
  146. [RANK: 4 ] get_file_data: (667749, 834685)
  147. [RANK: 2 ] merge: 333875-500811
  148. [RANK: 0 ] merge: 0-166937
  149. [RANK: 1 ] merge: 166938-333874
  150. [RANK: 3 ] merge: 500812-667748
  151. [RANK: 4 ] merge: 667749-834685
  152. [RANK: 3 ] ending: 3/5
  153. [RANK: 2 ] ending: 2/5
  154. [RANK: 0 ] ending: 0/5
  155. [RANK: 1 ] ending: 1/5
  156. [RANK: 4 ] ending: 4/5
  157. '''
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!