Python Compare two big lists

[simong1993]

Hey all,

So i have 2 huge lists. One is 3 million records. One is only 500k, What i need to do is compare 1 of the 3million records against the 500k and if its a 60% match or above link them. Here is my current code:-

from MasterFile import *
from difflib import SequenceMatcher

def GetAmazon():
    db_connection = mysql.connect(host=HOST, database="Amazon", user=USER, password=PASSWORD)
    cursor = db_connection.cursor()
    sql = "SELECT ID,Title,Ref,Found FROM Catalog"
    cursor.execute(sql)
    RawAmazon = cursor.fetchall()
    db_connection.commit()
    cursor.close()
    return RawAmazon

def GetEbay():
    db_connection = mysql.connect(host=HOST, database="Ebay", user=USER, password=PASSWORD)
    cursor = db_connection.cursor()
    sql = "SELECT ID,Title FROM Sold WHERE ASIN IS NULL ORDER BY RAND() LIMIT 5000"
    cursor.execute(sql)
    RawEbay = cursor.fetchall()
    db_connection.commit()
    cursor.close()
    return RawEbay

def Upload(EbayID,AmazonID,MatchPercent):
    print("Uploading",EbayID,AmazonID,MatchPercent)
    db_connection = mysql.connect(host=HOST, database="Amazon", user=USER, password=PASSWORD)
    cursor = db_connection.cursor()
    sql = "UPDATE Catalog SET Found = Found + 1 WHERE ID = %s LIMIT 1"
    val = (AmazonID,)
    cursor.execute(sql,val)
    db_connection.commit()
    cursor.close()

    db_connection = mysql.connect(host=HOST, database="Ebay", user=USER, password=PASSWORD)
    cursor = db_connection.cursor()
    sql = "UPDATE Sold SET ASIN = %s WHERE ID = %s LIMIT 1"
    val = (str(MatchPercent),str(EbayID),)
    cursor.execute(sql,val)
    db_connection.commit()
    cursor.close()
    print("Uploading End")


def UploadNoFound(EbayID):

    db_connection = mysql.connect(host=HOST, database="Ebay", user=USER, password=PASSWORD)
    cursor = db_connection.cursor()
    sql = "UPDATE Sold SET ASIN = %s WHERE ID = %s LIMIT 1"
    val = (0,EbayID)
    cursor.execute(sql,val)
    db_connection.commit()
    cursor.close()

def Checking(AmazonFull,Ebay):
    try:
        for A in AmazonFull:
            MatchPercent = SequenceMatcher(a=A[1], b=Ebay[1]).ratio()
            if MatchPercent > 0.60:
                Upload(Ebay[0],A[0],MatchPercent)
                Found = (A,Ebay,MatchPercent)
                print(Found)
                return Found
        UploadNoFound(Ebay[0])
        print("Not Found So Ending",Ebay[0])
    except:
        UploadNoFound(Ebay[0])
        print("Empty so ending",Ebay[0])


AmazonFull = GetAmazon()
EbayFull = GetEbay()

print(len(AmazonFull))
print(len(EbayFull))

while True:
    with concurrent.futures.ThreadPoolExecutor(max_workers=1000000000) as executor:
        # Start the load operations and mark each future with its URL
        future_to_url = {executor.submit(Checking, AmazonFull, Ebay): Ebay for Ebay in EbayFull}
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            print(url)
            try:
                data = future.result()
            except Exception as exc:
                err = ""
                print('%r generated an exception: %s' % (url, exc))
             #else:
             #   print('%r page is %d bytes' % (url, len(data)))

the issue is, its SO SO SO SLOW. Ive tried a Process Pool with no luck, ThreadPoolExecutor is okay but still slow. Is there a quicker way to do this *Looks at @Ghost *