Skip to content

Commit fcb500f

Browse files
reece394reece394
authored
Fix Firefox History Parsing and add name to Firefox Bookmarks (#44)
Fixes Firefox history parsing by changing the join id from moz_places.origin_id and moz_historyvisits.id to moz_places.id and moz_historyvisits.place_id. Additionally adds moz_historyvisits.visit_date which displays the unique visit date of each site every time it is visited rather than just the last visited timestamp. This also sorts by this field but retains the last visited timestamp as this is useful if a site is visited multiple times. Also adds a Firefox History Legacy definition which is functionally identical but drops the description and preview_image_url fields as several third party browsers as well as older Firefox versions do not have these fields and result in failure to parse otherwise. This also modifies the identify query to gate based on if these fields exist or not. This means there will only be one output rather than potentially two of the same database. Adds name to Firefox Bookmarks to help find it easier in the interface as well as spreadsheet exports.
1 parent 48ad6a0 commit fcb500f

5 files changed

Lines changed: 130 additions & 45 deletions

File tree

‎definitions/Firefox_Bookmarks.yaml‎

Lines changed: 3 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ Globs:
2020
- "{{MacOSFirefoxProfiles}}/*/places.sqlite"
2121

2222
Sources:
23-
- Preamble: |
24-
LET BookmarkTypes <= dict(`1`="URL", `2`="Folder", `3`="Separator")
23+
- name: Bookmarks
24+
Preamble: |
25+
LET BookmarkTypes <= dict(`1`='URL', `2`='Folder', `3`='Separator')
2526
2627
VQL: |
2728
SELECT ID, ParentID,
@@ -68,39 +69,3 @@ Sources:
6869
INNER JOIN moz_anno_attributes ON moz_annos.anno_attribute_id = moz_anno_attributes.id
6970
WHERE moz_anno_attributes.name IN ('downloads/destinationFileURI','downloads/destinationFileName','downloads/metaData')
7071
ORDER BY moz_annos.dateAdded ASC
71-
72-
- name: History
73-
Preamble: |
74-
LET VisitType <= dict(`1`='TRANSITION_LINK', `2`='TRANSITION_TYPED', `3`='TRANSITION_BOOKMARK',
75-
`4`='TRANSITION_EMBED', `5`= 'TRANSITION_REDIRECT_PERMANENT', `6`='TRANSITION_REDIRECT_TEMPORARY',
76-
`7`='TRANSITION_DOWNLOAD', `8`='TRANSITION_FRAMED_LINK', `9`='TRANSITION_RELOAD')
77-
78-
VQL: |
79-
SELECT VisitID, FromVisitID,
80-
timestamp(epoch= last_visit_date) AS LastVisitDate,
81-
VisitCount, URL, Title, Description,
82-
get(item= VisitType, field=str(str=visit_type), default="Unknown") AS VisitType,
83-
Bool(Value=hidden) AS Hidden,
84-
Bool(Value=typed) AS Typed,
85-
Frecency, PreviewImageURL, OSPath
86-
FROM Rows
87-
WHERE LastVisitDate > DateAfter AND LastVisitDate < DateBefore
88-
AND (Title, URL, Description) =~ FilterRegex
89-
90-
SQL: |
91-
SELECT
92-
moz_historyvisits.id AS VisitID,
93-
moz_historyvisits.from_visit AS FromVisitID,
94-
moz_places.last_visit_date,
95-
moz_places.visit_count AS VisitCount,
96-
moz_places.url AS URL,
97-
moz_places.title AS Title,
98-
moz_places.description AS Description,
99-
moz_historyvisits.visit_type,
100-
moz_places.hidden,
101-
moz_places.typed,
102-
moz_places.frecency AS Frecency,
103-
moz_places.preview_image_url AS PreviewImageURL
104-
FROM moz_places
105-
INNER JOIN moz_historyvisits ON moz_places.origin_id = moz_historyvisits.id
106-
ORDER BY moz_places.last_visit_date ASC

‎definitions/Firefox_History.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
Name: Firefox Places History
2+
Author: Andrew Rathbun, Reece394
3+
Email: andrew.d.rathbun@gmail.com
4+
Reference: https://github.com/EricZimmerman/SQLECmd
5+
SQLiteIdentifyQuery: |
6+
SELECT (
7+
SELECT COUNT(*)
8+
FROM sqlite_master
9+
WHERE type='table'
10+
AND (name='moz_historyvisits' OR name='moz_bookmarks'
11+
OR name='moz_places' OR name='moz_inputhistory')) +
12+
(SELECT CASE WHEN (SELECT COUNT(*) FROM pragma_table_info('moz_places') WHERE name IN ('description','preview_image_url')) > 1
13+
THEN 1 ELSE 0 END) AS `Check`;
14+
SQLiteIdentifyValue: 5
15+
Categories:
16+
- Firefox
17+
- Browser
18+
19+
FilenameRegex: "places.sqlite"
20+
Globs:
21+
- "{{WindowsFirefoxProfiles}}/*/places.sqlite"
22+
- "{{LinuxFirefoxProfiles}}/places.sqlite"
23+
- "{{MacOSFirefoxProfiles}}/*/places.sqlite"
24+
25+
Sources:
26+
- Preamble: |
27+
LET VisitType <= dict(`1`='TRANSITION_LINK', `2`='TRANSITION_TYPED', `3`='TRANSITION_BOOKMARK',
28+
`4`='TRANSITION_EMBED', `5`= 'TRANSITION_REDIRECT_PERMANENT', `6`='TRANSITION_REDIRECT_TEMPORARY',
29+
`7`='TRANSITION_DOWNLOAD', `8`='TRANSITION_FRAMED_LINK', `9`='TRANSITION_RELOAD')
30+
31+
VQL: |
32+
SELECT VisitID, FromVisitID,
33+
timestamp(epoch= visit_date) AS VisitDate,
34+
timestamp(epoch= last_visit_date) AS LastVisitDate,
35+
VisitCount, URL, Title, Description,
36+
get(item= VisitType, field=str(str=visit_type), default="Unknown") AS VisitType,
37+
Bool(Value=hidden) AS Hidden,
38+
Bool(Value=typed) AS Typed,
39+
Frecency, PreviewImageURL, OSPath
40+
FROM Rows
41+
WHERE VisitDate > DateAfter AND VisitDate < DateBefore
42+
AND (Title, URL, Description) =~ FilterRegex
43+
44+
SQL: |
45+
SELECT
46+
moz_historyvisits.id AS VisitID,
47+
moz_historyvisits.from_visit AS FromVisitID,
48+
moz_historyvisits.visit_date,
49+
moz_places.last_visit_date,
50+
moz_places.visit_count AS VisitCount,
51+
moz_places.url AS URL,
52+
moz_places.title AS Title,
53+
moz_places.description AS Description,
54+
moz_historyvisits.visit_type,
55+
moz_places.hidden,
56+
moz_places.typed,
57+
moz_places.frecency AS Frecency,
58+
moz_places.preview_image_url AS PreviewImageURL
59+
FROM moz_places
60+
INNER JOIN moz_historyvisits ON moz_places.id = moz_historyvisits.place_id
61+
ORDER BY moz_historyvisits.visit_date ASC

‎definitions/Firefox_HistoryLegacy.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
Name: Firefox Places History Legacy
2+
Author: Andrew Rathbun, Reece394
3+
Email: andrew.d.rathbun@gmail.com
4+
Reference: https://github.com/EricZimmerman/SQLECmd
5+
SQLiteIdentifyQuery: |
6+
SELECT (
7+
SELECT COUNT(*)
8+
FROM sqlite_master
9+
WHERE type='table'
10+
AND (name='moz_historyvisits' OR name='moz_bookmarks'
11+
OR name='moz_places' OR name='moz_inputhistory')) +
12+
(SELECT CASE WHEN (SELECT COUNT(*) FROM pragma_table_info('moz_places') WHERE name IN ('description','preview_image_url')) > 1
13+
THEN 0 ELSE 1 END) AS `Check`;
14+
SQLiteIdentifyValue: 5
15+
Categories:
16+
- Firefox
17+
- Browser
18+
19+
FilenameRegex: "places.sqlite"
20+
Globs:
21+
- "{{WindowsFirefoxProfiles}}/*/places.sqlite"
22+
- "{{LinuxFirefoxProfiles}}/places.sqlite"
23+
- "{{MacOSFirefoxProfiles}}/*/places.sqlite"
24+
25+
Sources:
26+
- Preamble: |
27+
LET VisitType <= dict(`1`='TRANSITION_LINK', `2`='TRANSITION_TYPED', `3`='TRANSITION_BOOKMARK',
28+
`4`='TRANSITION_EMBED', `5`= 'TRANSITION_REDIRECT_PERMANENT', `6`='TRANSITION_REDIRECT_TEMPORARY',
29+
`7`='TRANSITION_DOWNLOAD', `8`='TRANSITION_FRAMED_LINK', `9`='TRANSITION_RELOAD')
30+
31+
VQL: |
32+
SELECT VisitID, FromVisitID,
33+
timestamp(epoch= visit_date) AS VisitDate,
34+
timestamp(epoch= last_visit_date) AS LastVisitDate,
35+
VisitCount, URL, Title,
36+
get(item= VisitType, field=str(str=visit_type), default="Unknown") AS VisitType,
37+
Bool(Value=hidden) AS Hidden,
38+
Bool(Value=typed) AS Typed,
39+
Frecency, OSPath
40+
FROM Rows
41+
WHERE VisitDate > DateAfter AND VisitDate < DateBefore
42+
AND (Title, URL, Description) =~ FilterRegex
43+
44+
SQL: |
45+
SELECT
46+
moz_historyvisits.id AS VisitID,
47+
moz_historyvisits.from_visit AS FromVisitID,
48+
moz_historyvisits.visit_date,
49+
moz_places.last_visit_date,
50+
moz_places.visit_count AS VisitCount,
51+
moz_places.url AS URL,
52+
moz_places.title AS Title,
53+
moz_historyvisits.visit_type,
54+
moz_places.hidden,
55+
moz_places.typed,
56+
moz_places.frecency AS Frecency
57+
FROM moz_places
58+
INNER JOIN moz_historyvisits ON moz_places.id = moz_historyvisits.place_id
59+
ORDER BY moz_historyvisits.visit_date ASC

‎test_files/Firefox/firefox.sqlite‎

0 Bytes
Binary file not shown.

‎testing/fixtures/TestArtifact.golden‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,25 @@
55
" \"LastVisitDate\": \"2020-06-27T09:29:54.51375Z\",",
66
" \"URL\": \"https://www.mozilla.org/privacy/firefox/\",",
77
" \"Description\": null,",
8-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
8+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
99
" },",
1010
" {",
1111
" \"LastVisitDate\": \"2020-06-27T09:30:05.721357Z\",",
1212
" \"URL\": \"http://github.com/seanbreckenridge/dotfiles\",",
1313
" \"Description\": null,",
14-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
14+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
1515
" },",
1616
" {",
1717
" \"LastVisitDate\": \"2020-06-30T05:53:37.171Z\",",
1818
" \"URL\": \"https://www.mozilla.org/en-US/firefox/78.0a2/firstrun/\",",
1919
" \"Description\": \"Firefox Developer Edition is the blazing fast browser that offers cutting edge developer tools and latest features like CSS Grid support and framework debugging\",",
20-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
20+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
2121
" },",
2222
" {",
2323
" \"LastVisitDate\": \"2021-02-21T08:55:10.488Z\",",
2424
" \"URL\": \"https://www.mozilla.org/en-US/privacy/firefox/\",",
2525
" \"Description\": \"\\n Our Privacy Notices describe the data our products and services receive, share, and use, as well as choices available to you.\\n\",",
26-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
26+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
2727
" }",
2828
"]"
2929
],
@@ -33,7 +33,7 @@
3333
" \"LastVisitDate\": \"2021-02-21T08:55:10.488Z\",",
3434
" \"URL\": \"https://www.mozilla.org/en-US/privacy/firefox/\",",
3535
" \"Description\": \"\\n Our Privacy Notices describe the data our products and services receive, share, and use, as well as choices available to you.\\n\",",
36-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
36+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
3737
" }",
3838
"]"
3939
],
@@ -43,7 +43,7 @@
4343
" \"LastVisitDate\": \"2020-06-27T09:29:54.51375Z\",",
4444
" \"URL\": \"https://www.mozilla.org/privacy/firefox/\",",
4545
" \"Description\": null,",
46-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
46+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
4747
" }",
4848
"]"
4949
],
@@ -53,7 +53,7 @@
5353
" \"LastVisitDate\": \"2020-06-30T05:53:37.171Z\",",
5454
" \"URL\": \"https://www.mozilla.org/en-US/firefox/78.0a2/firstrun/\",",
5555
" \"Description\": \"Firefox Developer Edition is the blazing fast browser that offers cutting edge developer tools and latest features like CSS Grid support and framework debugging\",",
56-
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places_History\"",
56+
" \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
5757
" }",
5858
"]"
5959
]

0 commit comments

Comments
 (0)