-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathimplementation_of_rnns_and_lstms.py
More file actions
703 lines (415 loc) · 117 KB
/
implementation_of_rnns_and_lstms.py
File metadata and controls
703 lines (415 loc) · 117 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
# -*- coding: utf-8 -*-
"""Implementation of RNNs and LSTMs
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/1Sj6GcGYD3PfGg9E4TTZEDNx4TuH15rVD
# **RNN & LSTM**
# RNN (Recurrent Neural Network) & LSTM (Long Short-Term Memory)
<h2 style="font-family: 'poppins'; font-weight: bold; color: Green;">👨💻Author: Irfan Ullah Khan</h2>
[](https://github.com/programmarself)
[](https://www.kaggle.com/programmarself)
[](https://www.linkedin.com/in/irfan-ullah-khan-4a2871208/)
[](https://www.youtube.com/@irfanullahkhan7748)
[](mailto:programmarself@gmail.com)
[](https://datasciencetoyou.odoo.com)
# **Import Libraries**
"""
import numpy as np
import tensorflow as tf
import tensorflow.keras.models as models
import tensorflow.keras.layers as layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
"""# **Sample data**"""
# Sample data
sentences = [
"The cat slept on the windowsill, basking in the afternoon sun."
"She found a hidden note inside the old book."
"The coffee spilled, staining the tablecloth."
"The dog chased its tail in circles."
"An old man whistled a tune as he walked down the street."
"The cake was decorated with fresh strawberries and cream."
"He forgot his keys on the kitchen counter."
"The rain pitter-pattered against the rooftop all night."
"They laughed together, remembering their childhood adventures."
"The movie ended with a surprising plot twist."
"She wore a bright yellow scarf that caught everyone’s attention."
"The lighthouse stood tall against the stormy sea."
"He discovered a new hobby: collecting vintage postcards."
"The garden was full of blooming roses and lavender."
"The old clock on the wall ticked loudly in the quiet room."
"She hummed a melody while washing the dishes."
"The children built a fort out of blankets and pillows."
"A gentle breeze rustled the autumn leaves."
"He could hear the distant sound of a train whistle."
"The bookshop had a cozy reading nook by the fireplace."
"She danced gracefully across the stage in her new ballet shoes."
"The mountain view was breathtaking at sunrise."
"He made a wish while blowing out the candles on his birthday cake."
"The artist painted vibrant colors onto the canvas."
"The coffee shop had a welcoming aroma of freshly brewed beans."
"She found an old photograph in a dusty attic box."
"The wind chimes tinkled softly in the garden."
"He fixed the leaky faucet with a few twists of the wrench."
"The rainbows appeared after the storm had passed."
"The chef prepared a gourmet meal with exquisite precision."
"The little boat bobbed gently on the lake."
"She received a letter from an old friend she hadn’t seen in years."
"The concert was a spectacular display of lights and sound."
"The kitten played with a ball of yarn on the living room floor."
"The bakery window was filled with colorful pastries and cakes."
"He wore his favorite sweater even though it was warm outside."
"The stars shone brightly in the clear night sky."
"She sang a lullaby to her baby before bedtime."
"The wind carried the scent of fresh pine through the forest."
"The old man told stories of his youth to the fascinated children."
"She found a beautiful seashell on the beach."
"The library was silent except for the sound of turning pages."
"He sketched a portrait of his best friend in his notebook."
"The squirrel darted across the yard, clutching an acorn."
"The fog rolled in, enveloping the landscape in mystery."
"She practiced her guitar every evening on the porch."
"The fire crackled and popped in the fireplace."
"The road trip took them through stunning mountain passes."
"The old theater had ornate architecture and a grand chandelier."
"She carefully wrapped the gift with shiny paper and a bow."
"The snow covered the ground like a soft white blanket."
"He enjoyed a quiet afternoon of fishing by the river."
"The artist’s studio was cluttered with paintbrushes and canvases."
"The baby giggled as she played with her toys."
"The street was lined with charming cafes and boutique shops."
"He cooked a hearty stew on a cold winter’s day."
"The wind rustled the pages of the open book."
"The park was filled with the sounds of children playing."
"She enjoyed a relaxing bath with lavender-scented candles."
"The old barn had a rustic charm and a creaky door."
"The train journey offered scenic views of rolling hills."
"She wore a necklace that had been passed down through generations."
"The museum displayed ancient artifacts and priceless paintings."
"He looked out the window and saw a colorful hot air balloon."
"The scent of freshly cut grass filled the air."
"The dog barked excitedly at the sight of its owner."
"The recipe called for a pinch of salt and a dash of pepper."
"She admired the intricate design of the stained glass window."
"The ice cream melted quickly under the hot sun."
"The children gathered around the storyteller for a magical tale."
"The old map showed hidden treasures and forgotten lands."
"He planted seeds in the garden and hoped for a bountiful harvest."
"The bell rang, signaling the end of the school day."
"She decorated her room with colorful posters and fairy lights."
"The aroma of freshly baked bread wafted through the house."
"He enjoyed reading detective novels on rainy weekends."
"The picnic was complete with sandwiches, fruit, and lemonade."
"The city skyline looked majestic against the setting sun."
"She wrote a heartfelt letter to her future self."
"The fireflies twinkled like tiny stars in the evening dusk."
"The playground was filled with the laughter of children."
"He carefully folded the map and tucked it into his pocket."
"The clouds formed whimsical shapes in the blue sky."
"She found a cozy spot by the lake to read her book."
"The bicycle ride through the park was invigorating."
"The old piano in the corner was covered in dust but still beautiful."
"The garden gnome had a mischievous smile."
"The sand felt warm beneath her feet as she walked along the beach."
"The snowflakes fell gently, creating a winter wonderland."
"He practiced his magic tricks in front of the mirror."
"The castle on the hill looked majestic in the morning mist."
"She carefully painted her nails a vibrant shade of red."
"The sunset painted the sky with hues of orange and pink."
"The dog curled up next to the fire and fell asleep."
"The old library had towering shelves filled with books."
"She wore a hat decorated with colorful feathers."
"The melody of the song lingered in her mind throughout the day."
"The garden was a haven for birds and butterflies."
"He discovered an interesting rock while hiking in the mountains."
"The vintage car had been lovingly restored to its former glory."
]
"""# **Tokenization & Preprocessing of Data**"""
# Initialize the tokenizer
tokenizer = Tokenizer()
# Fit the tokenizer on the sentences
tokenizer.fit_on_texts(sentences)
# Get the total number of unique words (plus one for padding)
total_words = len(tokenizer.word_index) + 1
print(total_words)
print(tokenizer.word_index)
# Initialize a list to hold input sequences
input_sequences = []
# iterate over the sentences
for line in sentences:
# Convert the sentence into a sequence of integers
token_list = tokenizer.texts_to_sequences([line])[0]
# print(token_list)
# Create n-gram sequences from the sentence
for i in range(1, len(token_list)):
n_gram_sequence = token_list[:i+1]
input_sequences.append(n_gram_sequence)
# Determine the maximum length sequence
max_sequence_len = max([len(x) for x in input_sequences])
# Pad sequences to ensure they are all the same length
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))
print(input_sequences)
"""# **Data Preparation:**
"""
# Split the data into inputs and labels
# Inputs: All elements of the sequences except the one
X=input_sequences[:,:-1]
print("Input Data:",X)
# Labels: The last element each sequences
y=input_sequences[:,-1]
# print("Labels:",y)
# Convert the labels to one-hot encoding format
y = tf.keras.utils.to_categorical(y, num_classes=total_words)
# print("one hot encoded Vector: ",y)
"""# **Defining the Model Architecture:**
**Sequential Model:** A linear stack of layers./
**Embedding Layer:**/
**SimpleRNN Layer:**
"""
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, SimpleRNN, Dense
# ... (rest of the code remains the same)
# Define the RNN
# Sequential model allows stacking layers in a linear fashion
model = Sequential([Embedding(total_words, 10, input_length=max_sequence_len-1),
# SimpleRNN layer with 20 hidden units,which processes the sequence
SimpleRNN(30),
# Dense output layer with softmax activation function
# Output dimension: total number of words (for multi-class classification)
Dense(total_words, activation='softmax')
])
"""# **Compile the Model**"""
# Compile the model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
"""# **Train the Model**"""
# Train the model
model.fit(X, y, epochs=10, verbose=1)
"""# **Next Word Predication**
**Seed Text Converstion:**\
**Padding:** \
**Predication:**\
**Selection and Update:**\
**Iterative Predictioin:**
"""
# Function to predict the next word(s) given a seed text
def predict_next_word(seed_text, next_words=1):
for _ in range(next_words):
# Convert the seed text into a sequence of integers
token_list = tokenizer.texts_to_sequences([seed_text])[0]
# Pad the sequence to match the input length required by the model
token_list= pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
# Predict the probability distribution of the next word
predicted = model.predict(token_list, verbose=0)
# Get the index of the predicted word ,with the highest probability
predicted_word_index = np.argmax(predicted, axis=1)[0]
#Retrieve the word corresponding to the predicted index
predicted_word=tokenizer.index_word[predicted_word_index]
# Append the predicted word to the seed text
seed_text += " " + predicted_word
# Return the updated seed text with the predicted word
return seed_text
""""""
# Test the prediction function with a sample input
print(predict_next_word('Macine Learning'))
"""# **Building a Long Short-Term Memory(LSTM)**"""
# Import necessary libraries
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Embedding
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
"""# **Sample Data**"""
# Sample data
sentences = [
"The cat slept on the windowsill, basking in the afternoon sun."
"She found a hidden note inside the old book."
"The coffee spilled, staining the tablecloth."
"The dog chased its tail in circles."
"An old man whistled a tune as he walked down the street."
"The cake was decorated with fresh strawberries and cream."
"He forgot his keys on the kitchen counter."
"The rain pitter-pattered against the rooftop all night."
"They laughed together, remembering their childhood adventures."
"The movie ended with a surprising plot twist."
"She wore a bright yellow scarf that caught everyone’s attention."
"The lighthouse stood tall against the stormy sea."
"He discovered a new hobby: collecting vintage postcards."
"The garden was full of blooming roses and lavender."
"The old clock on the wall ticked loudly in the quiet room."
"She hummed a melody while washing the dishes."
"The children built a fort out of blankets and pillows."
"A gentle breeze rustled the autumn leaves."
"He could hear the distant sound of a train whistle."
"The bookshop had a cozy reading nook by the fireplace."
"She danced gracefully across the stage in her new ballet shoes."
"The mountain view was breathtaking at sunrise."
"He made a wish while blowing out the candles on his birthday cake."
"The artist painted vibrant colors onto the canvas."
"The coffee shop had a welcoming aroma of freshly brewed beans."
"She found an old photograph in a dusty attic box."
"The wind chimes tinkled softly in the garden."
"He fixed the leaky faucet with a few twists of the wrench."
"The rainbows appeared after the storm had passed."
"The chef prepared a gourmet meal with exquisite precision."
"The little boat bobbed gently on the lake."
"She received a letter from an old friend she hadn’t seen in years."
"The concert was a spectacular display of lights and sound."
"The kitten played with a ball of yarn on the living room floor."
"The bakery window was filled with colorful pastries and cakes."
"He wore his favorite sweater even though it was warm outside."
"The stars shone brightly in the clear night sky."
"She sang a lullaby to her baby before bedtime."
"The wind carried the scent of fresh pine through the forest."
"The old man told stories of his youth to the fascinated children."
"She found a beautiful seashell on the beach."
"The library was silent except for the sound of turning pages."
"He sketched a portrait of his best friend in his notebook."
"The squirrel darted across the yard, clutching an acorn."
"The fog rolled in, enveloping the landscape in mystery."
"She practiced her guitar every evening on the porch."
"The fire crackled and popped in the fireplace."
"The road trip took them through stunning mountain passes."
"The old theater had ornate architecture and a grand chandelier."
"She carefully wrapped the gift with shiny paper and a bow."
"The snow covered the ground like a soft white blanket."
"He enjoyed a quiet afternoon of fishing by the river."
"The artist’s studio was cluttered with paintbrushes and canvases."
"The baby giggled as she played with her toys."
"The street was lined with charming cafes and boutique shops."
"He cooked a hearty stew on a cold winter’s day."
"The wind rustled the pages of the open book."
"The park was filled with the sounds of children playing."
"She enjoyed a relaxing bath with lavender-scented candles."
"The old barn had a rustic charm and a creaky door."
"The train journey offered scenic views of rolling hills."
"She wore a necklace that had been passed down through generations."
"The museum displayed ancient artifacts and priceless paintings."
"He looked out the window and saw a colorful hot air balloon."
"The scent of freshly cut grass filled the air."
"The dog barked excitedly at the sight of its owner."
"The recipe called for a pinch of salt and a dash of pepper."
"She admired the intricate design of the stained glass window."
"The ice cream melted quickly under the hot sun."
"The children gathered around the storyteller for a magical tale."
"The old map showed hidden treasures and forgotten lands."
"He planted seeds in the garden and hoped for a bountiful harvest."
"The bell rang, signaling the end of the school day."
"She decorated her room with colorful posters and fairy lights."
"The aroma of freshly baked bread wafted through the house."
"He enjoyed reading detective novels on rainy weekends."
"The picnic was complete with sandwiches, fruit, and lemonade."
"The city skyline looked majestic against the setting sun."
"She wrote a heartfelt letter to her future self."
"The fireflies twinkled like tiny stars in the evening dusk."
"The playground was filled with the laughter of children."
"He carefully folded the map and tucked it into his pocket."
"The clouds formed whimsical shapes in the blue sky."
"She found a cozy spot by the lake to read her book."
"The bicycle ride through the park was invigorating."
"The old piano in the corner was covered in dust but still beautiful."
"The garden gnome had a mischievous smile."
"The sand felt warm beneath her feet as she walked along the beach."
"The snowflakes fell gently, creating a winter wonderland."
"He practiced his magic tricks in front of the mirror."
"The castle on the hill looked majestic in the morning mist."
"She carefully painted her nails a vibrant shade of red."
"The sunset painted the sky with hues of orange and pink."
"The dog curled up next to the fire and fell asleep."
"The old library had towering shelves filled with books."
"She wore a hat decorated with colorful feathers."
"The melody of the song lingered in her mind throughout the day."
"The garden was a haven for birds and butterflies."
"He discovered an interesting rock while hiking in the mountains."
"The vintage car had been lovingly restored to its former glory."
]
"""# **Tokenization & Preprocessing of Data**"""
# Initialize the tokenizer
tokenizer = Tokenizer()
# Fit the tokenizer on the sentences
tokenizer.fit_on_texts(sentences)
# Get the total number of unique words (plus one for padding)
total_words = len(tokenizer.word_index) + 1
print(total_words)
print(tokenizer.word_index)
# Initialize a list to hold input sequences
input_sequences = []
# iterate over the sentences
for line in sentences:
# Convert the sentence into a sequence of integers
token_list = tokenizer.texts_to_sequences([line])[0]
# print(token_list)
# Create n-gram sequences from the sentence
for i in range(1, len(token_list)):
n_gram_sequence = token_list[:i+1]
input_sequences.append(n_gram_sequence)
# Determine the maximum length sequence
max_sequence_len = max([len(x) for x in input_sequences])
# Pad sequences to ensure they are all the same length
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))
print(input_sequences)
"""# **Data Preparation:**
"""
# Split the data into inputs and labels
# Inputs: All elements of the sequences except the one
X=input_sequences[:,:-1]
print("Input Data:",X)
# Labels: The last element each sequences
y=input_sequences[:,-1]
# print("Labels:",y)
# Convert the labels to one-hot encoding format
y = tf.keras.utils.to_categorical(y, num_classes=total_words)
# print("one hot encoded Vector: ",y)
"""# **Defining the LSTM Model:**
"""
# Define the LSTM model
# Sequential Model allows stacking layers in a linear fashion
model = Sequential([
# Embedding layer to represent words as dense vectors
# Input dimension is the total number of unique words,Ouptut dimension:
# Input length is the length of the sequence (excluding the last word)
Embedding(total_words, 100, input_length=max_sequence_len-1),
# LSTM layer with 30 units
# LSTM (Long Shor-Term Memory) is a type of RNN that can capture long-term dependencies
LSTM(100),
# Dense output layer with a softmax activation function
# Output dimension is the total number of unique words
# Softmax activation converts the output to probabilities for each word
Dense(total_words, activation='softmax')
])
"""# **Compiling the Model**"""
# Compile the model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
"""# **Training the model**"""
# Train the model
model.fit(X, y, epochs=50, verbose=1)
"""# **Next Word Predication**
**Seed Text Converstion:**\
**Padding:** \
**Predication:**\
**Selection and Update:**\
**Iterative Predictioin:**
"""
# Function to predict the next word(s) given a seed text
def predict_next_word(seed_text, next_words=1): # Here if next_words=1 we will get only one word,if next_words=2 we will get 2 words
for _ in range(next_words):
# Convert the seed text into a sequence of integers
token_list = tokenizer.texts_to_sequences([seed_text])[0]
# Pad the sequence to match the input length required by the model
token_list= pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
# Predict the probability distribution of the next word
predicted = model.predict(token_list, verbose=0)
# Get the index of the predicted word ,with the highest probability
predicted_word_index = np.argmax(predicted, axis=1)[0]
#Retrieve the word corresponding to the predicted index
predicted_word=tokenizer.index_word[predicted_word_index]
# Append the predicted word to the seed text
seed_text += " " + predicted_word
# Return the updated seed text with the predicted word
return seed_text
"""# **Test prediction**"""
# Test the prediction function with a sample input
print(predict_next_word('She found a'))