0
votes

I am new to python. Need some help. i have a function which calls another functions and provide combined output. Function

def process_count(ldbtype, l_db_conn, local_df):
    global db_connection
    db_connection = l_db_conn
    
    local_df["schema_table"] = local_df["schema_name"] + "." + local_df["table_name"]

    local_df = local_df.replace(r'^\s*$', np.nan, regex=True)

    local_df["record_count"] = local_df["schema_table"].apply(record_count_check)
    local_df["pk_record_count"] = pk_record_count_check(local_df)
    local_df['delta_record_count'] = local_df.apply(
        lambda row: (row.record_count if pd.notnull(row.record_count) else 0) - (
            row.pk_record_count if pd.notnull(row.pk_record_count) else 0), axis=1)
    local_df["filter_column_cnt"] = filter_record_count_check(local_df)
    local_df["dups_cnt"] = dups_record_count_check(local_df)
    print(local_df)

    loging(datetime.datetime.now(), '-end', local_df)
    return (local_df)

output1:

   schema_name  table_name  pk_record_count  delta_record_count  filter_column_cnt  dups_cnt  
0   test_db    DimCurrency             3                   3                  2         2  
1   test_db  banking_fraud          1000                   0                208         0 

Function2:

def dupes_record(localdf):

    try:
        sql = 'SELECT * FROM ' + localdf["schema_table"] + ' WHERE ' + localdf["column_name"] + ' IN (SELECT ' + localdf["column_name"] + '\
               FROM ' + localdf["schema_table"] + ' group by ' + localdf["column_name"] + ' having count(*)>1)'
        df = pd.DataFrame()
        df["Records"] = sql[sql.notna()].apply(
            lambda i: pd.read_sql_query(i, db_connection))
        pd.set_option("display.max_colwidth", None)
        print(df)
        return df
    except Exception as e:
        exc_type, exc_obj, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        loging(datetime.datetime.now(), '-', sys.exc_info()[1])
        return np.nan

output2:

0     PersonID LastName FirstName     Address       City  Column1 Column2  Column3  
0         1  khyalia    sonali       delhi  new delhi       11     PAB      Balboa  
1         1  khyalia    sonali       delhi  new delhi       12     LVL      Latvian Lats
2         3    yadav    monika  vasantkunj      delhi       12     VEB      Bolivar  
3         3    yadav    monika  vasantkunj      delhi       12     VEB      Bolivar  
4         3    yadav    monika       noida         UP       15     VEB      Bolivar  

now my expectation is when output1 column "dups_cnt" is not zero or nan then output2 will become subdataframe of output1 column "dups_cnt".

So that when i convert to json then it should be like below example: note:have only convert 2 lines of output2 as example but it should pick full output2.

  {
    "table_name": "DimCurrency",
    "column_name": "PersonID",
    "business_key_name": "PersonID",
    "select_column_names": null,
    "where_condition": "FirstName ='sonali'",
    "schema_name": "testdb",
    "schema_table": "testdb.DimCurrency",
    "record_count": 6,
    "pk_record_count": 3,
    "delta_record_count": 3,
    "filter_column_cnt": 2,
    "dups_cnt&value": 2,{"PersonID":1,"LastName":"khyalia","FirstName":"sonali","Address":"delhi","City":"new delhi","Column1":11,"Column2":"PAB","Column3":"Balboa"},
{"PersonID":1,"LastName":"khyalia","FirstName":"sonali","Address":"delhi","City":"new delhi","Column1":12,"Column2":"LVL","Column3":"Latvian Lats"}
  },
{
    "table_name": "banking_fraud",
    "column_name": null,
    "business_key_name": "`index`",
    "select_column_names": null,
    "where_condition": "Gender='F'",
    "schema_name": "testdb",
    "schema_table": "testdb.banking_fraud",
    "record_count": 1000,
    "pk_record_count": 1000,
    "delta_record_count": 0,
    "filter_column_cnt": 208,
    "dups_cnt&value": 0,{"no duplicate record"}

  }