Analyzing Parallel Computing

Once again we will use image lab, this time to review Parallel Computing.

  • Change baseWidth in this line in code to increase computation requirements:def process_image(image, baseWidth=512): For instance 320, 512, 1024, 2048, 4096.- Compare Sequential and Parallel computing code and time to achieve outputs
from IPython.display import HTML, display
from pathlib import Path  # https://medium.com/@ageitgey/python-3-quick-tip-the-easy-way-to-deal-with-file-paths-on-windows-mac-and-linux-11a072b58d5f
from PIL import Image as pilImage # as PIL Image is used to avoid conflicts
from io import BytesIO
import base64
import numpy as np


# prepares a series of images
def image_data(path=Path("images/"), images=None):  # path of static images is defaulted
    if images is None:  # default image
        images = [
            {'source': "Internet", 'label': "Green Square", 'file': "green-square-16.png"},
            {'source': "Peter Carolin", 'label': "Clouds Impression", 'file': "clouds-impression.png"},
            {'source': "Peter Carolin", 'label': "Lassen Volcano", 'file': "lassen-volcano.jpg"}
        ]
    for image in images:
        # File to open
        image['filename'] = path / image['file']  # file with path
    return images

# Scale to baseWidth
def scale_image(img, baseWidth):
    scalePercent = (baseWidth/float(img.size[0]))
    scaleHeight = int((float(img.size[1])*float(scalePercent)))
    scale = (baseWidth, scaleHeight)
    return img.resize(scale)

# PIL image converted to base64
def image_to_base64(img, format):
    with BytesIO() as buffer:
        img.save(buffer, format)
        return base64.b64encode(buffer.getvalue()).decode()
    
# Convert pixels to Grey Scale
def grey_pixel(pixel):
    average = (pixel[0] + pixel[1] + pixel[2]) // 3  # average pixel values and use // for integer division
    if len(pixel) > 3:
        return( (average, average, average, pixel[3]) ) # PNG format
    else:
        return( (average, average, average) )
    
# Convert pixels to Red Scale
def red_pixel(pixel):
    if len(pixel) > 3:
        return( (pixel[0], 0, 0, pixel[3]) ) # PNG format
    else:
        return( (pixel[0], 0, 0) )
    
# Convert pixels to Red Scale
def green_pixel(pixel):
    if len(pixel) > 3:
        return( (0, pixel[1], 0, pixel[3]) ) # PNG format
    else:
        return( (0, pixel[1], 0) )
    
# Convert pixels to Red Scale
def blue_pixel(pixel):
    if len(pixel) > 3:
        return( (0, 0, pixel[2], pixel[3]) ) # PNG format
    else:
        return( (0, 0, pixel[2]) )
        
# Set Properties of Image, Scale, and convert to Base64
def image_management(image, baseWidth):  # path of static images is defaulted        
    # Image open return PIL image object
    img = pilImage.open(image['filename'])
    
    # Python Image Library operations
    image['format'] = img.format
    image['mode'] = img.mode
    image['size'] = img.size
    # Scale the Image
    img = scale_image(img, baseWidth)
    image['pil'] = img
    image['scaled_size'] = img.size
    image['numpy'] = np.array(img.getdata())
    # Scaled HTML
    image['html'] = '<img src="data:image/png;base64,%s">' % image_to_base64(image['pil'], image['format'])
    
    # Grey HTML
    # each pixel in numpy array is turned to grey 
    # then resulting list, using List Comprehension, is put back into img    
    img.putdata([grey_pixel(pixel) for pixel in image['numpy']])
    image['html_grey'] =  '<img src="data:image/png;base64,%s">' % image_to_base64(img, image['format'])
    
    # Red HTML
    img.putdata([red_pixel(pixel) for pixel in image['numpy']])
    image['html_red'] =  '<img src="data:image/png;base64,%s">' % image_to_base64(img, image['format'])
    
    # Green HTML
    img.putdata([green_pixel(pixel) for pixel in image['numpy']])
    image['html_green'] =  '<img src="data:image/png;base64,%s">' % image_to_base64(img, image['format'])
    
    # Blue HTML
    img.putdata([blue_pixel(pixel) for pixel in image['numpy']])
    image['html_blue'] =  '<img src="data:image/png;base64,%s">' % image_to_base64(img, image['format'])
    
    
def process_image(image, baseWidth=320):
    image_management(image, baseWidth)
    print("---- meta data -----")
    print(image['label'])
    print(image['source'])
    print(image['format'])
    print(image['mode'])
    print("Original size: ", image['size'])
    print("Scaled size: ", image['scaled_size'])
    
    print("-- images --")
    display(HTML(image['html'])) 
    display(HTML(image['html_grey'])) 
    display(HTML(image['html_red'])) 
    display(HTML(image['html_green'])) 
    display(HTML(image['html_blue'])) 

Sequential Processing

The for loop iterates over the list of images and processes them one at a time, in order.

if __name__ == "__main__":
    # setup default images
    images = image_data()

    # Sequential Processing    
    for image in images:
        process_image(image)
        
    print()
---- meta data -----
Green Square
Internet
PNG
RGBA
Original size:  (16, 16)
Scaled size:  (320, 320)
-- images --
---- meta data -----
Clouds Impression
Peter Carolin
PNG
RGBA
Original size:  (320, 234)
Scaled size:  (320, 234)
-- images --
---- meta data -----
Lassen Volcano
Peter Carolin
JPEG
RGB
Original size:  (2792, 2094)
Scaled size:  (320, 240)
-- images --

Parallel Computing

In parallel or concurrent mode, the ThreadPoolExecutor is used to submit each image to a separate worker thread, allowing multiple images to be processed simultaneously. Multithreading allows multiple concurrent tasks of a process at the same time. The executor.map() method is used to apply the process_image function to each image in the images list.

  • The order in which the images are processed is not guaranteed, as threads are performed simultaneously.
import concurrent.futures

# Jupyter Notebook Visualization of Images
if __name__ == "__main__":
    # setup default images
    images = image_data()
    
    # Parallel Processsing
    # executor allocates threads, it considers core execution capability of machine
    with concurrent.futures.ThreadPoolExecutor() as executor:
        executor.map(process_image, images)  # order is not predictable
        
    print()
---- meta data -----
Green Square
Internet
PNG
RGBA
Original size:  (16, 16)
Scaled size:  (320, 320)
-- images --
---- meta data -----
Clouds Impression
Peter Carolin
PNG
RGBA
Original size:  (320, 234)
Scaled size:  (320, 234)
-- images --
---- meta data -----
Lassen Volcano
Peter Carolin
JPEG
RGB
Original size:  (2792, 2094)
Scaled size:  (320, 240)
-- images --

Observing Parallel Computing and Threads

You can observe Processes, CPU Percentage, and Threads with Tools on your machine. Common tools to monitor performance are Activity Monitor on MacOS or Task Manager on Windows.

  • This example is using top launched in VSCode Terminal. (mac)
  • Try top -H for linux.
    • PID is Process ID.
    • COMMAND is task running on machine. Python is activated when running this Jupyter notebook.
    • #TH is number of threads. This increases from 15/1 to 18/1 on my machine when running python parallel computing example.

Hacks

AP Classroom. Provide answers and thoughts on theoritical question form college board Video in section 4.3. They start at about the 9 minute mark.

Example 1

In order to run 2 processes at the same time, this is how to the processes would run:

1st processor- 10 seconds, 30 seconds

2nd processor- 50 seconds

Therefore, the maximum would be 50 seconds.

Example 2

Running parallel: 25 second one finishes than 20 seconds still needed to finish the 45 second one.

Total 45 seconds

Running on 1 processor: 25+45=70 seconds

Data Structures. Build a List Comprehension example

  • list = [calc(item) for item in items]
midnights = [
    "Lavender Haze",
    "Maroon",
    "Anti-Hero",
    "Snow on the Beach",
    "You're on Your Own Kids",
    "Midnight Rain",
    "Question...?",
    "Vigilante Sh*t",
    "Bejeweled",
    "Labyrinth",
     "Karma",
    "Sweet Nothing",
    "Mastermind",
    "The Great War",
    "Bigger Than The Whole Sky",
    "Paris",
    "High Infidelity",
    "Glitch",
    "Would've, Could've, Should've",
    "Dear Reader"
]
newlist = [x for x in midnights if len(x) < 10]
print(newlist)
['Maroon', 'Anti-Hero', 'Bejeweled', 'Labyrinth', 'Karma', 'Paris', 'Glitch']
tracks_with_t = [x for x in midnights if "t" in x]
print(tracks_with_t)
['Anti-Hero', 'Snow on the Beach', 'Midnight Rain', 'Question...?', 'Vigilante Sh*t', 'Labyrinth', 'Sweet Nothing', 'Mastermind', 'The Great War', 'High Infidelity', 'Glitch']
uppercase = [x.upper() for x in midnights]   #make a new list with all uppercase 
print(uppercase)
['LAVENDER HAZE', 'MAROON', 'ANTI-HERO', 'SNOW ON THE BEACH', "YOU'RE ON YOUR OWN KIDS", 'MIDNIGHT RAIN', 'QUESTION...?', 'VIGILANTE SH*T', 'BEJEWELED', 'LABYRINTH', 'KARMA', 'SWEET NOTHING', 'MASTERMIND', 'THE GREAT WAR', 'BIGGER THAN THE WHOLE SKY', 'PARIS', 'HIGH INFIDELITY', 'GLITCH', "WOULD'VE, COULD'VE, SHOULD'VE", 'DEAR READER']
lowercase = [x.lower() for x in uppercase]   #make a new list with all uppercase 
print(lowercase)
['lavender haze', 'maroon', 'anti-hero', 'snow on the beach', "you're on your own kids", 'midnight rain', 'question...?', 'vigilante sh*t', 'bejeweled', 'labyrinth', 'karma', 'sweet nothing', 'mastermind', 'the great war', 'bigger than the whole sky', 'paris', 'high infidelity', 'glitch', "would've, could've, should've", 'dear reader']
num = [1, 2, 3]
tuple_list = [(x, y) for x in midnights for y in num]
print(tuple_list)
[('Lavender Haze', 1), ('Lavender Haze', 2), ('Lavender Haze', 3), ('Maroon', 1), ('Maroon', 2), ('Maroon', 3), ('Anti-Hero', 1), ('Anti-Hero', 2), ('Anti-Hero', 3), ('Snow on the Beach', 1), ('Snow on the Beach', 2), ('Snow on the Beach', 3), ("You're on Your Own Kids", 1), ("You're on Your Own Kids", 2), ("You're on Your Own Kids", 3), ('Midnight Rain', 1), ('Midnight Rain', 2), ('Midnight Rain', 3), ('Question...?', 1), ('Question...?', 2), ('Question...?', 3), ('Vigilante Sh*t', 1), ('Vigilante Sh*t', 2), ('Vigilante Sh*t', 3), ('Bejeweled', 1), ('Bejeweled', 2), ('Bejeweled', 3), ('Labyrinth', 1), ('Labyrinth', 2), ('Labyrinth', 3), ('Karma', 1), ('Karma', 2), ('Karma', 3), ('Sweet Nothing', 1), ('Sweet Nothing', 2), ('Sweet Nothing', 3), ('Mastermind', 1), ('Mastermind', 2), ('Mastermind', 3), ('The Great War', 1), ('The Great War', 2), ('The Great War', 3), ('Bigger Than The Whole Sky', 1), ('Bigger Than The Whole Sky', 2), ('Bigger Than The Whole Sky', 3), ('Paris', 1), ('Paris', 2), ('Paris', 3), ('High Infidelity', 1), ('High Infidelity', 2), ('High Infidelity', 3), ('Glitch', 1), ('Glitch', 2), ('Glitch', 3), ("Would've, Could've, Should've", 1), ("Would've, Could've, Should've", 2), ("Would've, Could've, Should've", 3), ('Dear Reader', 1), ('Dear Reader', 2), ('Dear Reader', 3)]