File tree Expand file tree Collapse file tree 21 files changed +28
-39
lines changed Expand file tree Collapse file tree 21 files changed +28
-39
lines changed Original file line number Diff line number Diff line change @@ -106,6 +106,8 @@ target/
106
106
107
107
# mkdocs
108
108
site
109
- -h
110
- logs
109
+
110
+ # airflow
111
+ sqlite /
112
+ logs /
111
113
simple_auth_manager_passwords.json.generated
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import Variable , dag , task
7
8
from app import discord
8
- from airflow .sdk import dag
9
- from airflow .sdk import task
10
- from airflow .sdk import Variable
11
9
12
10
DEFAULT_ARGS = {
13
11
"owner" : "David Jr." ,
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import Variable , dag , task
7
8
from app import discord
8
9
from app .finance_bot .udf import (
9
10
df_difference ,
12
13
refine_diff_df_to_string ,
13
14
write_to_bigquery ,
14
15
)
15
- from airflow .sdk import dag
16
- from airflow .sdk import task
17
- from airflow .sdk import Variable
18
16
19
17
DEFAULT_ARGS = {
20
18
"owner" : "CHWan" ,
Original file line number Diff line number Diff line change 3
3
import numpy as np
4
4
import pandas as pd
5
5
import pygsheets
6
- from google .cloud import bigquery
7
6
from airflow .sdk import Variable
7
+ from google .cloud import bigquery
8
8
9
9
10
10
def df_difference (df_xls : pd .DataFrame , df_bigquery : pd .DataFrame ) -> pd .DataFrame :
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import Variable , dag , task
7
8
from app import discord
8
9
from app .proposal_reminder .udf import get_proposal_summary
9
- from airflow .sdk import dag
10
- from airflow .sdk import task
11
- from airflow .sdk import Variable
12
10
13
11
DEFAULT_ARGS = {
14
12
"owner" : "Henry Lee" ,
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import Variable , dag , task
7
8
from app import discord
8
9
from app .team_registration_bot .udf import (
9
10
_compose_discord_msg ,
10
11
_get_statistics_from_bigquery ,
11
12
)
12
- from airflow .sdk import dag
13
- from airflow .sdk import task
14
- from airflow .sdk import Variable
15
13
16
14
DEFAULT_ARGS = {
17
15
"owner" : "David Jr." ,
Original file line number Diff line number Diff line change 6
6
7
7
import requests
8
8
from airflow import settings
9
+ from airflow .sdk import Variable , dag , task
9
10
from sqlalchemy .orm import sessionmaker
10
- from airflow .sdk import dag
11
- from airflow .sdk import task
12
- from airflow .sdk import Variable
13
11
14
12
DEFAULT_ARGS = {
15
13
"owner" : "David Jr." ,
Original file line number Diff line number Diff line change 1
1
"""
2
2
Scrape Facebook posts and insights data, save to BigQuery
3
3
"""
4
+
4
5
from datetime import datetime , timedelta
5
6
7
+ from airflow .sdk import dag , task
6
8
from utils .posts_insights .facebook import FacebookPostsInsightsParser
7
- from airflow .sdk import dag
8
- from airflow .sdk import task
9
9
10
10
DEFAULT_ARGS = {
11
11
"owner" : "CHWan" ,
Original file line number Diff line number Diff line change 1
1
"""
2
2
Scrape Instagram posts and insights data, save to BigQuery
3
3
"""
4
+
4
5
from datetime import datetime , timedelta
5
6
7
+ from airflow .sdk import dag , task
6
8
from utils .posts_insights .instagram import InstagramPostsInsightsParser
7
- from airflow .sdk import dag
8
- from airflow .sdk import task
9
9
10
10
DEFAULT_ARGS = {
11
11
"owner" : "Angus Yang" ,
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import dag , task
7
8
from ods .kktix_ticket_orders .udfs import bigquery_loader , kktix_api
8
- from airflow .sdk import dag
9
- from airflow .sdk import task
10
9
11
10
DEFAULT_ARGS = {
12
11
"owner" : "David Jr." ,
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import dag , task
7
8
from ods .kktix_ticket_orders .udfs import kktix_refund
8
- from airflow .sdk import dag
9
- from airflow .sdk import task
10
9
11
10
DEFAULT_ARGS = {
12
11
"owner" : "Henry Lee" ,
Original file line number Diff line number Diff line change 4
4
import requests
5
5
import tenacity
6
6
from airflow .providers .http .hooks .http import HttpHook
7
+ from airflow .sdk import Variable
7
8
from dateutil .parser import parse
8
9
from ods .kktix_ticket_orders .udfs import kktix_loader , kktix_transformer
9
- from airflow .sdk import Variable
10
10
11
11
SCHEDULE_INTERVAL_SECONDS : int = 3600
12
12
HTTP_HOOK = HttpHook (http_conn_id = "kktix_api" , method = "GET" )
Original file line number Diff line number Diff line change 1
1
"""
2
2
Scrape LinkedIn posts and insights data, save to BigQuery
3
3
"""
4
+
4
5
from datetime import datetime , timedelta
5
6
7
+ from airflow .sdk import dag , task
6
8
from utils .posts_insights .linkedin import LinkedinPostsInsightsParser
7
- from airflow .sdk import dag
8
- from airflow .sdk import task
9
9
10
10
DEFAULT_ARGS = {
11
11
"owner" : "Angus Yang" ,
Original file line number Diff line number Diff line change 1
1
"""
2
2
Scrape X (Twitter) posts and insights data, save to BigQuery
3
3
"""
4
+
4
5
from datetime import datetime , timedelta
5
6
7
+ from airflow .sdk import dag , task
6
8
from utils .posts_insights .twitter import TwitterPostsInsightsParser
7
- from airflow .sdk import dag
8
- from airflow .sdk import task
9
9
10
10
DEFAULT_ARGS = {
11
11
"owner" : "Henry Lee" ,
Original file line number Diff line number Diff line change 4
4
5
5
from datetime import datetime , timedelta
6
6
7
+ from airflow .sdk import dag , task
7
8
from ods .youtube .udfs import youtube_api
8
- from airflow .sdk import dag
9
- from airflow .sdk import task
10
9
11
10
DEFAULT_ARGS = {
12
11
"owner" : "David Jr." ,
Original file line number Diff line number Diff line change 5
5
import pandas as pd
6
6
from airflow import macros
7
7
from airflow .providers .http .hooks .http import HttpHook
8
+ from airflow .sdk import Variable
8
9
from google .cloud import bigquery
9
10
from utils .hook_related import RETRY_ARGS
10
- from airflow .sdk import Variable
11
11
12
12
# channel id of YouTube is public to everyone, so it's okay to commit this ID into git
13
13
CHANNEL_ID = "UCHLnNgRnfGYDzPCCH8qGbQw"
Original file line number Diff line number Diff line change 2
2
from datetime import datetime
3
3
4
4
import requests
5
+ from airflow .sdk import Variable
5
6
from google .cloud import bigquery
6
7
from utils .posts_insights .base import (
7
8
CREATE_POST_TABLE_SQL_TEMPLATE ,
8
9
BasePostsInsightsParser ,
9
10
)
10
- from airflow .sdk import Variable
11
11
12
12
logger = logging .getLogger (__name__ )
13
13
Original file line number Diff line number Diff line change 2
2
from datetime import datetime
3
3
4
4
import requests
5
+ from airflow .sdk import Variable
5
6
from utils .posts_insights .base import (
6
7
CREATE_INSIGHT_TABLE_SQL_TEMPLATE ,
7
8
CREATE_POST_TABLE_SQL_TEMPLATE ,
8
9
BasePostsInsightsParser ,
9
10
)
10
- from airflow .sdk import Variable
11
11
12
12
logger = logging .getLogger (__name__ )
13
13
Original file line number Diff line number Diff line change 2
2
from datetime import datetime
3
3
4
4
import requests
5
+ from airflow .sdk import Variable
5
6
from utils .posts_insights .base import (
6
7
CREATE_INSIGHT_TABLE_SQL_TEMPLATE ,
7
8
CREATE_POST_TABLE_SQL_TEMPLATE ,
8
9
BasePostsInsightsParser ,
9
10
)
10
- from airflow .sdk import Variable
11
11
12
12
logger = logging .getLogger (__name__ )
13
13
Original file line number Diff line number Diff line change 2
2
from datetime import datetime
3
3
4
4
import requests
5
+ from airflow .sdk import Variable
5
6
from utils .posts_insights .base import (
6
7
CREATE_INSIGHT_TABLE_SQL_TEMPLATE ,
7
8
CREATE_POST_TABLE_SQL_TEMPLATE ,
8
9
BasePostsInsightsParser ,
9
10
)
10
- from airflow .sdk import Variable
11
11
12
12
logger = logging .getLogger (__name__ )
13
13
Original file line number Diff line number Diff line change 1
1
import pytest
2
2
3
3
4
- @pytest .fixture ()
4
+ @pytest .fixture
5
5
def kktix_api_data ():
6
6
return [
7
7
{
You can’t perform that action at this time.
0 commit comments