Support lineage of Pandas.DataFrame

> I have uses XLX file and 3 parqute files as source and performed some teansformation. the code ran good and i could able to see the linegae in spline. but i could able see only 3 parqute files as source , and xlsx source is not displying in linegae .

> the above is the code i used and i have attached the screenshots of databricks cluster and libraries i have installed on cluster level. Can you get back with me on this. Thankyou in advance.

_Originally posted by @harishyadavdevops in https://github.com/AbsaOSS/spline-spark-agent/issues/262#issuecomment-1519417615_

```python
# COMMAND ----------
#from pyspark.sql import SparkSession
from pyspark.sql.types import IntegerType
from pyspark.sql.functions import col,to_date,datediff,current_date

import pandas
import xlrd

#from pyspark.sql.types import *

#spark=SparkSession.builder\
# .appName("POC For Lineage")\
# .config("spark.jars.packages","com.crealytics:spark-excel_2.12:0.13.7")\
# .getOrCreate()

input_filepath='/dbfs/FileStore'
input_filepaths='dbfs:/FileStore'

pandas_df=pandas.read_excel(f'{input_filepath}/CMO_ERICA_AIM_SAP_Mapping_Master_Latest.xlsx',index_col=None)

df_cmo_master=spark.createDataFrame(pandas_df)\
.select( \
col("IDERICA"), \
col("TargetDays").alias("target_days"), \
col("PrimaryPlatformPlan").alias("plan_platfrom"), \
col("sitename").alias("cmo_site"), \
col("primaryplatform").alias("pes_platform"), \
) \
.distinct()

df_cmo_master.show()
#df_cmo_master = spark.sql('Select 1 as IDERICA, 1 as Account_Number')

df_qrta = spark.read.format('parquet').option("header",True).load(f'{input_filepaths}/qrta_intermediate.parquet')
df_external_supplier = spark.read.format('parquet').option("header",True).load(f'{input_filepaths}/external_supplier.parquet')
df_comet_supplier= spark.read.format('parquet').option("header",True).load(f'{input_filepaths}/comet_supplier.parquet')

# dbfs:/FileStore/comet_supplier.parquet
# dbfs:/FileStore/external_supplier.parquet
# dbfs:/FileStore/qrta_intermediate.parquet
# dbfs:/FileStore/CMO_ERICA_AIM_SAP_Mapping_Master_Latest.xlsx

df_qrta_intermediate = df_qrta.select(
col("PO - Batch_ID").alias("po_batch_id"),
col("Batch").alias("vendor_batch_id"),
col("SKU").alias("sku"),
col("SKU Description").alias("sku_description"),
col("Brand Name").alias("sku_brand"),
col("Brand Platform").alias("sku_brand_platform"),
to_date(col("Janssen Release - Actual"),'MM-dd-yyyy').alias("janssen_release_date"),
to_date(col("Batch Start - Actual"),'MM-dd-yyyy').alias("manufacturing_start_date"),
to_date(col("Batch End - Actual"),'MM-dd-yyyy').alias("manufacturing_completion_date"),
to_date(col("CMO Release - Actual"),'MM-dd-yyyy').alias("cmo_release_date"),
datediff(
to_date(col("janseen_release"),'MM-dd-yyyy'), to_date(col("batch_end"),'MM-dd-yyyy')
).alias("release_turn_around_time_overall"),
datediff(
to_date(col("cmo_actual"),'MM-dd-yyyy'), to_date(col("batch_end"),'MM-dd-yyyy')
).alias("release_turn_around_time_cmo"),
datediff(
to_date(col("janseen_release"),'MM-dd-yyyy'), to_date(col("cmo_actual"),'MM-dd-yyyy')
).alias("release_turn_around_time_janssen"),
col("# Returns to CMO").cast("int").alias("count_of_iteration_needed"),
col("Status – Final Release").alias("final_release_status"),
to_date(col("Status Date Final Releasee"),'MM-dd-yyyy').alias("final_release_status_date"),
col("CMO Site ID").alias("cmo_id")
)

# COMMAND ----------

df_qrta_final = (
df_qrta_intermediate.alias("qrta_intermediate")
.join(
df_cmo_master.alias("cmo_master"),
col("qrta_intermediate.cmo_id") == col("cmo_master.IDERICA"),"left"
)
.join(
df_external_supplier.alias("external_supplier"),
col("qrta_intermediate.cmo_id") ==col("external_supplier.Site ID (Legacy ERICA ID)"),"left"
)
.join(
df_comet_supplier.alias("comet_supplier"),
col("external_supplier.COMET ID") == col("comet_supplier.Account Number"),"left"
)
.select(
"qrta_intermediate.*",
"cmo_master.cmo_site",
"cmo_master.plan_platfrom",
"cmo_master.pes_platform",
col("cmo_master.target_days").cast("int").alias("target"),
col("comet_supplier.Account Name").alias("quality_cmo_site"),
col("comet_supplier.Responsible Owning Group").alias("eq_platform"),
current_date().alias("last_refresh_date")

)
)

df_qrta_final.show()

```