与 Google BigQuery 交互
读写BigQuery数据库,需要额外依赖项:
$ pip install google-cloud-bigquery
读取
从BigQuery查询并得到DataFrame
,可以像这样:
import polars as pl
from google.cloud import bigquery
client = bigquery.Client()
# 执行查询
QUERY = (
'SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` '
'WHERE state = "TX" '
'LIMIT 100')
query_job = client.query(QUERY) # API 请求
rows = query_job.result() # 等待查询完成
df = pl.from_arrow(rows.to_arrow())
写入
from google.cloud import bigquery
client = bigquery.Client()
with io.BytesIO() as stream:
df.write_parquet(stream)
stream.seek(0)
job = client.load_table_from_file(
stream,
destination='tablename',
project='projectname',
job_config=bigquery.LoadJobConfig(
source_format=bigquery.SourceFormat.PARQUET,
),
)
job.result()