I am new to python. Need some help. i have a function which calls another functions and provide combined output. Function
def process_count(ldbtype, l_db_conn, local_df):
global db_connection
db_connection = l_db_conn
local_df["schema_table"] = local_df["schema_name"] + "." + local_df["table_name"]
local_df = local_df.replace(r'^\s*$', np.nan, regex=True)
local_df["record_count"] = local_df["schema_table"].apply(record_count_check)
local_df["pk_record_count"] = pk_record_count_check(local_df)
local_df['delta_record_count'] = local_df.apply(
lambda row: (row.record_count if pd.notnull(row.record_count) else 0) - (
row.pk_record_count if pd.notnull(row.pk_record_count) else 0), axis=1)
local_df["filter_column_cnt"] = filter_record_count_check(local_df)
local_df["dups_cnt"] = dups_record_count_check(local_df)
print(local_df)
loging(datetime.datetime.now(), '-end', local_df)
return (local_df)
output1:
schema_name table_name pk_record_count delta_record_count filter_column_cnt dups_cnt
0 test_db DimCurrency 3 3 2 2
1 test_db banking_fraud 1000 0 208 0
Function2:
def dupes_record(localdf):
try:
sql = 'SELECT * FROM ' + localdf["schema_table"] + ' WHERE ' + localdf["column_name"] + ' IN (SELECT ' + localdf["column_name"] + '\
FROM ' + localdf["schema_table"] + ' group by ' + localdf["column_name"] + ' having count(*)>1)'
df = pd.DataFrame()
df["Records"] = sql[sql.notna()].apply(
lambda i: pd.read_sql_query(i, db_connection))
pd.set_option("display.max_colwidth", None)
print(df)
return df
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
loging(datetime.datetime.now(), '-', sys.exc_info()[1])
return np.nan
output2:
0 PersonID LastName FirstName Address City Column1 Column2 Column3
0 1 khyalia sonali delhi new delhi 11 PAB Balboa
1 1 khyalia sonali delhi new delhi 12 LVL Latvian Lats
2 3 yadav monika vasantkunj delhi 12 VEB Bolivar
3 3 yadav monika vasantkunj delhi 12 VEB Bolivar
4 3 yadav monika noida UP 15 VEB Bolivar
now my expectation is when output1 column "dups_cnt" is not zero or nan then output2 will become subdataframe of output1 column "dups_cnt".
So that when i convert to json then it should be like below example: note:have only convert 2 lines of output2 as example but it should pick full output2.
{
"table_name": "DimCurrency",
"column_name": "PersonID",
"business_key_name": "PersonID",
"select_column_names": null,
"where_condition": "FirstName ='sonali'",
"schema_name": "testdb",
"schema_table": "testdb.DimCurrency",
"record_count": 6,
"pk_record_count": 3,
"delta_record_count": 3,
"filter_column_cnt": 2,
"dups_cnt&value": 2,{"PersonID":1,"LastName":"khyalia","FirstName":"sonali","Address":"delhi","City":"new delhi","Column1":11,"Column2":"PAB","Column3":"Balboa"},
{"PersonID":1,"LastName":"khyalia","FirstName":"sonali","Address":"delhi","City":"new delhi","Column1":12,"Column2":"LVL","Column3":"Latvian Lats"}
},
{
"table_name": "banking_fraud",
"column_name": null,
"business_key_name": "`index`",
"select_column_names": null,
"where_condition": "Gender='F'",
"schema_name": "testdb",
"schema_table": "testdb.banking_fraud",
"record_count": 1000,
"pk_record_count": 1000,
"delta_record_count": 0,
"filter_column_cnt": 208,
"dups_cnt&value": 0,{"no duplicate record"}
}