Блог

Мне нужно извлечь информацию из базы данных SQL Server, используя ETL из AWS GLUE с помощью Python, это то, что я пытаюсь сделать

 import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from pyspark.sql import SQLContext
from awsglue.dynamicframe import DynamicFrame
import pymssql

args = getResolvedOptions(sys.argv, ['JOB_NAME'])
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

dbread = pymssql.connect('Host', 'user', 'pass', 'DB')
cursorread = dbread.cursor()
cursorread.execute("select tra.codigo_tramo, pozI.descripcion   ' - '   pozF.descripcion, ope.razon_social as nombre_transportador 
                    from m_tramo tra, m_pozo pozI, m_pozo pozF, m_operador ope 
                    where tra.estado <> 'I' and tra.codigo_pozo_ini = pozI.codigo_pozo 
                    and tra.codigo_pozo_fin = pozF.codigo_pozo 
                    and tra.codigo_trasportador = ope.codigo_operador")
                    
dbread.close()
consulta = spark.createDataFrame(consulta, ['codigo_tramo', 'descripcion_tramo','codigo_transportador'])
consulta.show()
  

Как я могу ознакомиться с этой информацией?

Я уже решил это

 import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext, SparkConf
from awsglue.context import GlueContext
from awsglue.job import Job
from pyspark.sql import SQLContext
from awsglue.dynamicframe import DynamicFrame

args = getResolvedOptions(sys.argv, ['JOB_NAME'])
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

# Construct JDBC connection options
connection_sqlserver_options = {
    "url": "",
    "dbtable": "",
    "user": "",
    "password": ""}
    
# Read from JDBC databases with custom driver
DyF = glueContext.create_dynamic_frame.from_options(connection_type="sqlserver",
                                                          connection_options=connection_sqlserver_options)