11import csv
2+ import io
23import logging
34import time
5+ import uuid
6+ from pathlib import Path
7+ from typing import Optional
8+ from typing import Tuple
49
510from django .core .management .base import BaseCommand
611from django .db .models import Exists
914from django .db .models import Q
1015from django .db .models .expressions import F
1116from django_cte import With
17+ from le_utils .constants import content_kinds
1218
1319from contentcuration .models import Channel
1420from contentcuration .models import ContentNode
21+ from contentcuration .models import License
1522
1623
1724logger = logging .getLogger (__name__ )
1825
1926
27+ class LicensingFixesLookup (object ):
28+ """Consolidates logic for reading and processing the licensing fixes from the CSV"""
29+
30+ def __init__ (self ):
31+ self ._lookup = {}
32+ self ._license_lookup = {}
33+
34+ def load (self , fp : io .TextIOWrapper ):
35+ """Loads the data from the CSV file, and the necessary license data from the database"""
36+ reader = csv .DictReader (fp )
37+ license_names = set ()
38+
39+ # create a lookup index by channel ID from the CSV data
40+ for row in reader :
41+ lookup_key = f"{ uuid .UUID (row ['channel_id' ]).hex } { row .get ('kind' , '' )}
42+ self ._lookup [lookup_key ] = row
43+ if row ["license_name" ]:
44+ license_names .add (row ["license_name" ])
45+
46+ # load all licenses, regardless of whether they are named in the CSV
47+ license_lookup_by_name = {}
48+ for lic in License .objects .all ():
49+ self ._license_lookup [lic .id ] = lic
50+ license_lookup_by_name [lic .license_name ] = lic
51+ license_names .discard (lic .license_name )
52+
53+ # ensure we've found all the licenses
54+ if len (license_names ):
55+ raise ValueError (f"Could not find all licenses: { license_names } )
56+
57+ # we now are certain all licenses are found
58+ for info in self ._lookup .values ():
59+ if info ["license_name" ]:
60+ info ["license_id" ] = license_lookup_by_name [info ["license_name" ]].id
61+
62+ def get_info (
63+ self ,
64+ channel_id : str ,
65+ kind : str ,
66+ license_id : Optional [int ],
67+ license_description : Optional [str ],
68+ copyright_holder : Optional [str ],
69+ ) -> Tuple [Optional [int ], Optional [str ], Optional [str ]]:
70+ """
71+ Determines the complete licensing metadata, given the current metadata, and comparing it
72+ with what would make the node complete.
73+
74+ :param channel_id: The channel the node was sourced from
75+ :param kind: The content kind of the node
76+ :param license_id: The current license_id of the node
77+ :param license_description: The current license_description of the node
78+ :param copyright_holder: The current copyright_holder of the node
79+ :return: A tuple of (license_id, license_description, copyright_holder) to use on the node
80+ """
81+ # first check kind-specific metadata, fallback to channel-wide (no kind)
82+ info = self ._lookup .get (f"{ channel_id } { kind } , None )
83+ if info is None :
84+ info = self ._lookup .get (f"{ channel_id } , None )
85+
86+ if info is None :
87+ logger .warning (f"Failed to find licensing info for channel: { channel_id } )
88+ return license_id , license_description , copyright_holder
89+
90+ if not license_id :
91+ license_id = info ["license_id" ]
92+
93+ if not license_id :
94+ return None , license_description , copyright_holder
95+
96+ license_obj = self ._license_lookup .get (license_id )
97+
98+ if license_obj .is_custom and not license_description :
99+ license_description = info ["license_description" ]
100+
101+ if license_obj .copyright_holder_required and not copyright_holder :
102+ copyright_holder = info ["copyright_holder" ]
103+
104+ return license_id , license_description , copyright_holder
105+
106+
20107class Command (BaseCommand ):
21108 """
22109 Audits nodes that have imported content from public channels and whether the imported content
23- has a missing source node.
24-
25- TODO: this does not yet FIX them
110+ has a missing source node. We've determined that pretty much all of these have incomplete
111+ licensing data
26112 """
27113
28114 def handle (self , * args , ** options ):
@@ -71,32 +157,27 @@ def handle(self, *args, **options):
71157
72158 logger .info ("=== Iterating over private destination channels. ===" )
73159 channel_count = 0
74- total_node_count = 0
75-
76- with open ("fix_missing_import_sources.csv" , "w" , newline = "" ) as csv_file :
77- csv_writer = csv .DictWriter (
78- csv_file ,
79- fieldnames = [
80- "channel_id" ,
81- "channel_name" ,
82- "contentnode_id" ,
83- "contentnode_title" ,
84- "public_channel_id" ,
85- "public_channel_name" ,
86- "public_channel_deleted" ,
87- ],
88- )
89- csv_writer .writeheader ()
160+ total_fixed = 0
161+ lookup = LicensingFixesLookup ()
162+
163+ command_dir = Path (__file__ ).parent
164+ csv_path = command_dir / "licensing_fixes_lookup.csv"
165+
166+ with csv_path .open ("r" , encoding = "utf-8" , newline = "" ) as csv_file :
167+ lookup .load (csv_file )
90168
91- for channel in destination_channels .iterator ():
92- node_count = self .handle_channel (csv_writer , channel )
169+ # skip using an iterator here, to limit transaction duration to `handle_channel`
170+ for channel in destination_channels :
171+ node_count = self .handle_channel (lookup , channel )
93172
94- if node_count > 0 :
95- total_node_count += node_count
96- channel_count += 1
173+ if node_count > 0 :
174+ total_fixed += node_count
175+ channel_count += 1
97176
98177 logger .info ("=== Done iterating over private destination channels. ===" )
99- logger .info (f"Found { total_node_count } { channel_count } )
178+ logger .info (
179+ f"Fixed incomplete licensing data on { total_fixed } { channel_count }
180+ )
100181 logger .info (f"Finished in { time .time () - start } )
101182
102183 def get_public_cte (self ) -> With :
@@ -110,7 +191,15 @@ def get_public_cte(self) -> With:
110191 name = "public_cte" ,
111192 )
112193
113- def handle_channel (self , csv_writer : csv .DictWriter , channel : dict ) -> int :
194+ def handle_channel (self , lookup : LicensingFixesLookup , channel : dict ) -> int :
195+ """
196+ Goes through the nodes of the channel, that were imported from public channels, but no
197+ longer have a valid source node. For each node, it applies license metadata as necessary
198+
199+ :param lookup: The lookup utility to pull licensing data from
200+ :param channel: The channel to fix
201+ :return: The total node count that are now marked complete as a result of the fixes
202+ """
114203 public_cte = self .get_public_cte ()
115204 channel_id = channel ["id" ]
116205 channel_name = channel ["name" ]
@@ -127,6 +216,7 @@ def handle_channel(self, csv_writer: csv.DictWriter, channel: dict) -> int:
127216 public_channel_name = public_cte .col .name ,
128217 public_channel_deleted = public_cte .col .deleted ,
129218 )
219+ .exclude (kind = content_kinds .TOPIC )
130220 .filter (
131221 Q (public_channel_deleted = True )
132222 | ~ Exists (
@@ -136,29 +226,51 @@ def handle_channel(self, csv_writer: csv.DictWriter, channel: dict) -> int:
136226 )
137227 )
138228 )
139- .values (
140- "public_channel_id" ,
141- "public_channel_name" ,
142- "public_channel_deleted" ,
143- contentnode_id = F ("id" ),
144- contentnode_title = F ("title" ),
145- )
146229 )
147230
148231 # Count and log results
149232 node_count = missing_source_nodes .count ()
233+ processed = 0
234+ was_complete = 0
235+ unfixed = 0
236+ now_complete = 0
150237
151- # TODO: this will be replaced with logic to correct the missing source nodes
152- if node_count > 0 :
238+ def _log ():
153239 logger .info (
154- f"{ channel_id } { channel_name } \t { node_count } node(s) with missing source nodes. "
240+ f"Fixing { channel_id } { channel_name } \t total: { node_count } ; before: { was_complete } unfixed: { unfixed } ; after: { now_complete } ; "
155241 )
156- row_dict = {
157- "channel_id" : channel_id ,
158- "channel_name" : channel_name ,
159- }
160- for node_dict in missing_source_nodes .iterator ():
161- row_dict .update (node_dict )
162- csv_writer .writerow (row_dict )
163-
164- return node_count
242+
243+ if node_count > 0 :
244+ for node in missing_source_nodes .iterator ():
245+ # determine the new license metadata
246+ license_id , license_description , copyright_holder = lookup .get_info (
247+ node .original_channel_id ,
248+ node .kind_id ,
249+ node .license_id ,
250+ node .license_description ,
251+ node .copyright_holder ,
252+ )
253+
254+ # if there isn't a license, there's nothing to do
255+ if not license_id :
256+ unfixed += 1
257+ # cannot fix
258+ continue
259+
260+ if node .complete :
261+ was_complete += 1
262+
263+ # apply updates
264+ node .license_id = license_id
265+ node .license_description = license_description
266+ node .copyright_holder = copyright_holder
267+ if not node .mark_complete ():
268+ now_complete += 1
269+ node .save ()
270+ processed += 1
271+ if processed % 100 == 0 :
272+ _log ()
273+
274+ _log ()
275+
276+ return now_complete - was_complete
0 commit comments