' Copyright (C) 2023 Andy https://github.com/AAndyProgram

Imports System.Net

Imports System.Threading

Imports PersonalUtilities.Functions.RegularExpressions

Imports PersonalUtilities.Functions.XML

Imports PersonalUtilities.Tools.ImageRenderer

Imports PersonalUtilities.Tools.Web.Clients

Imports PersonalUtilities.Tools.Web.Clients.Base

Imports PersonalUtilities.Tools.Web.Documents.JSON

Imports SCrawler.API.Base

Imports SCrawler.API.Reddit.RedditViewExchange

Imports SCrawler.API.YouTube.Objects

Imports SCrawler.Plugin.Hosts

Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period

Imports CView = SCrawler.API.Reddit.IRedditView.View

Imports UStates = SCrawler.API.Base.UserMedia.States

Imports UTypes = SCrawler.API.Base.UserMedia.Types

Namespace API.Reddit

Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView

#Region "Declarations"

Private Const CannelsLabelName As String = "Channels"

Friend Const CannelsLabelName_ChannelsForm As String = "RChannels"

Private ReadOnly Property MySiteSettings As SiteSettings

Get

Return DirectCast(HOST.Source, SiteSettings)

End Get

End Property

Private ReadOnly Property DateTrueProvider(ByVal IsChannel As Boolean) As IFormatProvider

Get

Return UnixDate32Provider

'Return If(IsChannel, UnixDate32ProviderReddit, UnixDate64Provider)

End Get

End Property

Private ReadOnly Property UseM3U8 As Boolean

Get

Return Settings.UseM3U8 And CBool(DirectCast(HOST.Source, SiteSettings).UseM3U8.Value)

End Get

End Property

Friend Property IsChannel As Boolean = False

Friend Overrides ReadOnly Property SpecialLabels As IEnumerable(Of String)

Get

Return {CannelsLabelName, CannelsLabelName_ChannelsForm, UserLabelName}

End Get

End Property

Private _RedGifsAccount As String = String.Empty

Friend Property RedGifsAccount As String Implements IRedditView.RedGifsAccount

Get

If Not _RedGifsAccount.IsEmptyString Then

Return _RedGifsAccount

ElseIf Not ChannelInfo Is Nothing Then

Return ChannelInfo.RedGifsAccount

Else

Return String.Empty

End If

End Get

Set(ByVal acc As String)

_RedGifsAccount = acc

End Set

End Property

Private _RedditAccount As String = String.Empty

Friend Property RedditAccount As String Implements IRedditView.RedditAccount

Get

If IsChannelForm Then

Return _RedditAccount

Else

Return MyBase.AccountName

End If

End Get

Set(ByVal acc As String)

_RedditAccount = acc

End Set

End Property

Friend Overrides Property AccountName As String

Get

Return RedditAccount

End Get

Set(ByVal acc As String)

MyBase.AccountName = acc

End Set

End Property

#End Region

#Region "Channels Support"

#Region "IChannelLimits Support"

Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount

Friend Property DownloadLimitPost As String Implements IChannelLimits.DownloadLimitPost

Friend Property DownloadLimitDate As Date? Implements IChannelLimits.DownloadLimitDate

Friend Overloads Sub SetLimit(Optional ByVal MaxPost As String = "", Optional ByVal MaxCount As Integer? = Nothing,

Optional ByVal MinDate As Date? = Nothing) Implements IChannelLimits.SetLimit

DownloadLimitPost = MaxPost

DownloadLimitCount = MaxCount

DownloadLimitDate = MinDate

End Sub

Friend Overloads Sub SetLimit(ByVal Source As IChannelLimits) Implements IChannelLimits.SetLimit

With Source

DownloadLimitCount = .DownloadLimitCount

DownloadLimitPost = .DownloadLimitPost

DownloadLimitDate = .DownloadLimitDate

AutoGetLimits = .AutoGetLimits

End With

End Sub

Friend Property AutoGetLimits As Boolean = True Implements IChannelLimits.AutoGetLimits

Private ReadOnly Property IsChannelForm As Boolean

Get

Return Not IsSavedPosts AndAlso IsChannel AndAlso Not ChannelInfo Is Nothing

End Get

End Property

#End Region

Friend Property ChannelInfo As Channel

Private ReadOnly ChannelPostsNames As List(Of String)

Friend Property SkipExistsUsers As Boolean = False

Private ReadOnly _ExistsUsersNames As List(Of String)

Friend Property SaveToCache As Boolean = False

Friend Function GetNewChannelPosts() As IEnumerable(Of UserPost)

If _ContentNew.Count > 0 Then Return (From c As UserMedia In _ContentNew

Where Not c.Post.CachedFile.IsEmptyString And c.State = UStates.Downloaded

Select c.Post) Else Return Nothing

End Function

#End Region

#Region "IRedditView Support"

Friend Property ViewMode As CView Implements IRedditView.ViewMode

Friend Property ViewPeriod As CPeriod Implements IRedditView.ViewPeriod

Friend Sub SetView(ByVal Options As IRedditView) Implements IRedditView.SetView

If Not Options Is Nothing Then

With Options

ViewMode = .ViewMode

ViewPeriod = .ViewPeriod

DownloadText = .DownloadText

DownloadTextPosts = .DownloadTextPosts

DownloadTextSpecialFolder = .DownloadTextSpecialFolder

RedGifsAccount = .RedGifsAccount

RedditAccount = .RedditAccount

If TypeOf Options Is RedditViewExchange Then DirectCast(Options, RedditViewExchange).Apply(Me)

End With

End If

End Sub

Private ReadOnly Property View As String

Get

Select Case ViewMode

Case CView.Hot : Return "hot"

Case CView.Top : Return "top"

Case CView.Best : Return "best"

Case CView.Rising : Return "rising"

Case Else : Return "new"

End Select

End Get

End Property

Private ReadOnly Property Period As String

Get

If ViewMode = CView.Top Then

Select Case ViewPeriod

Case CPeriod.Hour : Return "hour"

Case CPeriod.Day : Return "day"

Case CPeriod.Week : Return "week"

Case CPeriod.Month : Return "month"

Case CPeriod.Year : Return "year"

Case Else : Return "all"

End Select

Else

Return "all"

End If

End Get

End Property

Friend Overrides Property DownloadText As Boolean Implements IRedditView.DownloadText

Friend Overrides Property DownloadTextPosts As Boolean Implements IRedditView.DownloadTextPosts

Friend Overrides Property DownloadTextSpecialFolder As Boolean Implements IRedditView.DownloadTextSpecialFolder

#End Region

#Region "Initializer"

Friend Sub New()

ChannelPostsNames = New List(Of String)

_ExistsUsersNames = New List(Of String)

_CrossPosts = New List(Of String)

UseMD5Comparison = True

StartMD5Checked = True

RemoveExistingDuplicates = False

UseInternalDownloadFileFunction = True

UseInternalM3U8Function = True

End Sub

#End Region

#Region "Load and Update user info"

Private Function UpdateNames() As Boolean

If NameTrue(True).IsEmptyString Then

Dim n$() = Name.Split("@")

If n.ListExists Then

If n.Length = 2 Then

NameTrue = n(0)

IsChannel = True

ElseIf IsChannel Then

NameTrue = Name

Else

NameTrue = n(0)

End If

End If

If Not IsSavedPosts Then

Dim l$ = IIf(IsChannel, CannelsLabelName, UserLabelName)

Settings.Labels.Add(l)

Labels.ListAddValue(l, LNC)

Labels.Sort()

Return True

End If

End If

Return False

End Function

Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)

With Container

If Loading Then

ViewMode = .Value(Name_ViewMode).FromXML(Of Integer)(CInt(CView.New))

ViewPeriod = .Value(Name_ViewPeriod).FromXML(Of Integer)(CInt(CPeriod.All))

IsChannel = .Value(Name_IsChannel).FromXML(Of Boolean)(False)

RedGifsAccount = .Value(Name_RedGifsAccount)

RedditAccount = .Value(Name_RedditAccount)

UpdateNames()

Else

If UpdateNames() Then .Value(Name_LabelsName) = LabelsString

.Add(Name_ViewMode, CInt(ViewMode))

.Add(Name_ViewPeriod, CInt(ViewPeriod))

.Add(Name_IsChannel, IsChannel.BoolToInteger)

.Add(Name_TrueName, NameTrue(True))

.Add(Name_RedGifsAccount, RedGifsAccount)

.Add(Name_RedditAccount, RedditAccount)

End If

End With

End Sub

Friend Overrides Function ExchangeOptionsGet() As Object

Return New RedditViewExchange(Me)

End Function

Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)

If Not Obj Is Nothing AndAlso TypeOf Obj Is IRedditView Then SetView(DirectCast(Obj, IRedditView))

End Sub

#End Region

#Region "Download Overrides"

Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)

Err429Count = 0

_CrossPosts.Clear()

If CreatedByChannel And Settings.FromChannelDownloadTopUse And Settings.FromChannelDownloadTop > 0 Then _

DownloadTopCount = Settings.FromChannelDownloadTop.Value

If IsChannel Or IsSavedPosts Then UseMD5Comparison = False

If IsSavedPosts Then NameTrue = MySiteSettings.SavedPostsUserName.Value

UpdateNames()

If IsChannelForm Then

UseMD5Comparison = False

EnvirDownloadSet()

If Not Responser Is Nothing Then Responser.Dispose()

Responser = New Responser

Responser.Copy(MySiteSettings.Responser)

ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)

If Not ViewMode = CView.New Then ChannelPostsNames.ListAddList(ChannelInfo.PostsNames, LNC)

If SkipExistsUsers Then _ExistsUsersNames.ListAddList(Settings.UsersList.Select(Function(p) p.Name), LNC)

DownloadDataF(Token)

ReparseVideo(Token)

_ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)

DownloadContent(Token)

Else

MyBase.DownloadData(Token)

End If

End Sub

Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)

With MySiteSettings

If IsSavedPosts Then

If Not CBool(.UseTokenForSavedPosts.Value) Then Responser.Headers.Remove(DeclaredNames.Header_Authorization)

Else

If Not CBool(.UseCookiesForTimelines.Value) Then Responser.Cookies.Clear()

If Not CBool(.UseTokenForTimelines.Value) Then Responser.Headers.Remove(DeclaredNames.Header_Authorization)

End If

End With

Responser.ProcessExceptionDecision = AddressOf Err429Process

_TotalPostsDownloaded = 0

If IsSavedPosts Then

Responser.DecodersError = EDP.ReturnValue

DownloadDataChannel(String.Empty, Token)

ElseIf IsChannel Then

If ChannelInfo Is Nothing Then

ChannelPostsNames.ListAddList(_TempPostsList, LNC)

If ChannelPostsNames.Count > 0 Then

DownloadLimitCount = Nothing

With _ContentList.Where(Function(c) c.Post.Date.HasValue)

If .Count > 0 Then DownloadLimitDate = .Max(Function(p) p.Post.Date.Value).AddMinutes(-10)

End With

End If

If DownloadTopCount.HasValue Then DownloadLimitCount = DownloadTopCount

Else

GetUserInfo()

End If

If SaveToCache Then

DownloadText = False

DownloadTextPosts = False

DownloadTextSpecialFolder = False

If Not Responser.Decoders.Contains(SymbolsConverter.Converters.HTML) Then _

Responser.Decoders.Add(SymbolsConverter.Converters.HTML)

End If

DownloadDataChannel(String.Empty, Token)

If ChannelInfo Is Nothing Then _TempPostsList.ListAddList(_TempMediaList.Select(Function(m) m.Post.ID), LNC)

Else

GetUserInfo()

DownloadDataUser(String.Empty, Token)

End If

ProgressPre.Done()

End Sub

#End Region

#Region "Download Functions (User, Channel)"

Private Err429Count As Integer = 0

Private Err429TryAgain As Boolean = False

Private _TotalPostsDownloaded As Integer = 0

Private ReadOnly _CrossPosts As List(Of String)

Private Const SiteGfycatKey As String = "gfycat"

Private Const SiteRedGifsKey As String = "redgifs"

Private Const Node_CrosspostRootId As String = "crosspostRootId"

Private Const Node_CrosspostParentId As String = "crosspostParentId"

Private Const Node_CrosspostParent As String = "crosspost_parent"

Private Sub Wait429()

With MySiteSettings

If Not Err429TryAgain Then .RequestCount += 1

Err429TryAgain = False

If (.RequestCount Mod 100) = 0 Then Thread.Sleep(60100)

End With

End Sub

Private Function Err429Process(ByVal Status As IResponserStatus, ByVal NullArg As Object, ByVal CurrErr As ErrorsDescriber) As ErrorsDescriber

If Not Status Is Nothing AndAlso Status.StatusCode = 429 Then

If Err429Count = 0 Then

Err429Count += 1

MySiteSettings.RequestCount = 100

Err429TryAgain = True

Return EDP.ReturnValue

End If

End If

Return CurrErr

End Function

Private Sub Err429Reset()

Err429Count = 0

Err429TryAgain = False

End Sub

Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)

Dim eObj% = 0

Dim round% = 0

Dim URL$ = String.Empty

Dim _completed As Boolean = False

Do

round += 1

Try

Dim PostID$ = String.Empty, PostTmp$ = String.Empty

Dim PostDate$

Dim n As EContainer, nn As EContainer

Dim NewPostDetected As Boolean = False

Dim ExistsDetected As Boolean = False

Dim IsCrossPost As Predicate(Of EContainer) = Function(e) Not e.Value(Node_CrosspostRootId).IsEmptyString Or Not e.Value(Node_CrosspostParentId).IsEmptyString Or Not e.Value(Node_CrosspostParent).IsEmptyString

Dim CheckNode As Predicate(Of EContainer) = Function(e) Not ParseUserMediaOnly OrElse If(e("author")?.Value, "/").ToLower.Equals(NameTrue.StringToLower)

Dim _PostID As Func(Of String) = Function() PostTmp.IfNullOrEmpty(PostID)

'URL = $"https://gateway.reddit.com/desktopapi/v1/user/{NameTrue}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"

URL = $"https://oauth.reddit.com/user/{NameTrue}/submitted.json?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"

ThrowAny(Token)

Wait429()

Dim r$ = Responser.GetResponse(URL)

If Not r.IsEmptyString Then

Err429Reset()

Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing

If w.Count > 0 Then

'n = w.GetNode(JsonNodesJson)

n = w.GetNode(ChannelJsonNodes)

If n.ListExists Then

ProgressPre.ChangeMax(n.Count)

For Each nn In n

ProgressPre.Perform()

ThrowAny(Token)

With nn("data")

If .ListExists Then

If CheckNode(.Self) Then

'Obtain post ID

PostID = String.Empty

PostTmp = .Value("name") '.Name

If PostTmp.IsEmptyString Then PostTmp = .Value("id")

If PostTmp.IsEmptyString Then Continue For

'Check for CrossPost

If IsCrossPost(.Self) Then

_CrossPosts.ListAddList({ .Value(Node_CrosspostRootId),

.Value(Node_CrosspostParentId),

.Value(Node_CrosspostParent),

PostTmp}, LNC)

If ParseUserMediaOnly Then Continue For

Else

If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty

End If

'Download decision

If Not _TempPostsList.Contains(_PostID()) Then

NewPostDetected = True

_TempPostsList.Add(_PostID())

Else

If Not _CrossPosts.Contains(_PostID()) Then ExistsDetected = True

Continue For

End If

PostDate = If(.Item("created")?.Value, String.Empty)

Select Case CheckDatesLimit(PostDate, DateTrueProvider(IsChannel))

Case DateResult.Skip : Continue For

Case DateResult.Exit : Exit Sub

End Select

ParseContainer(.Self, _PostID(), PostDate,,, GetTextDocument(.Self))

End If

End If

End With

Next

End If

End If

End Using

If POST.IsEmptyString And ExistsDetected Then Exit Sub

If Not _PostID().IsEmptyString And NewPostDetected Then DownloadDataUser(_PostID(), Token)

ElseIf Err429TryAgain Then

Continue Do

End If

_completed = True

Catch ex As Exception

If ProcessException(ex, Token, $"data downloading error [{URL}]",, eObj) = HttpStatusCode.InternalServerError Then

If round = 2 Then eObj = HttpStatusCode.InternalServerError

Else

_completed = True

End If

End Try

Loop While Not _completed

End Sub

Private Sub DownloadDataChannel(ByVal POST As String, ByVal Token As CancellationToken)

Const savedPostsSleepTimer% = 2000

Dim eObj% = 0

Dim round% = 0

Dim URL$ = String.Empty

Dim _completed As Boolean = False

Do

round += 1

Try

Dim PostID$ = String.Empty

Dim PostDate$, _UserID$

Dim n As EContainer, nn As EContainer, s As EContainer

Dim NewPostDetected As Boolean = False

Dim ExistsDetected As Boolean = False

Dim eCount As Predicate(Of EContainer) = Function(e) e.Count > 0

Dim lDate As Date?

If IsSavedPosts Then

URL = $"https://www.reddit.com/user/{NameTrue}/saved.json?after={POST}"

If Not POST.IsEmptyString Then Thread.Sleep(savedPostsSleepTimer)

Else

URL = $"https://reddit.com/r/{NameTrue}/{View}.json?allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"

End If

ThrowAny(Token)

Wait429()

Dim r$ = Responser.GetResponse(URL)

'If IsSavedPosts Then Err429Count = 0

If Not r.IsEmptyString Then

Err429Reset()

Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing

If w.Count > 0 Then

n = w.GetNode(ChannelJsonNodes)

If Not n Is Nothing AndAlso n.Count > 0 Then

ProgressPre.ChangeMax(n.Count)

For Each nn In n

ProgressPre.Perform()

ThrowAny(Token)

s = nn.ItemF({eCount})

If If(s?.Count, 0) > 0 Then

PostID = s.Value("name")

If PostID.IsEmptyString AndAlso s.Contains("id") Then PostID = s("id").Value

If ChannelPostsNames.Contains(PostID) Then

If ViewMode = CView.New Then ExistsDetected = True Else NewPostDetected = True 'bypass

Continue For

End If

If DownloadLimitCount.HasValue AndAlso _TotalPostsDownloaded >= DownloadLimitCount.Value Then Exit Sub

If Not DownloadLimitPost.IsEmptyString AndAlso DownloadLimitPost = PostID Then Exit Sub

If ViewMode = CView.New AndAlso DownloadLimitDate.HasValue AndAlso _TempMediaList.Count > 0 Then

With (From __u In _TempMediaList Where __u.Post.Date.HasValue Select __u.Post.Date.Value)

If .Count > 0 Then lDate = .Min Else lDate = Nothing

End With

If lDate.HasValue AndAlso lDate.Value <= DownloadLimitDate.Value Then Exit Sub

End If

If IsSavedPosts Then

If Not _TempPostsList.Contains(PostID) Then

NewPostDetected = True

_TempPostsList.Add(PostID)

Else

ExistsDetected = True

Continue For

End If

Else

NewPostDetected = True

End If

If s.Contains("created") Then PostDate = s("created").Value Else PostDate = String.Empty

_UserID = s.Value("author")

If Not IsSavedPosts AndAlso SkipExistsUsers AndAlso _ExistsUsersNames.Count > 0 AndAlso

Not _UserID.IsEmptyString AndAlso _ExistsUsersNames.Contains(_UserID) Then

If Not IsSavedPosts AndAlso Not ChannelInfo Is Nothing Then _

ChannelInfo.ChannelExistentUserNames.ListAddValue(_UserID, LNC)

Continue For

End If

ParseContainer(s, PostID, PostDate, _UserID,, If(Not SaveToCache, GetTextDocument(s), String.Empty))

End If

Next

End If

End If

End Using

If POST.IsEmptyString And ExistsDetected Then Exit Sub

If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token)

ElseIf Err429TryAgain Then

Continue Do

End If

_completed = True

Catch ex As Exception

Dim errValue% = ProcessException(ex, Token, $"{IIf(IsSavedPosts, "saved posts", "channel")} data downloading error [{URL}]",, eObj)

If errValue = HttpStatusCode.InternalServerError Then

If round = 2 Then eObj = HttpStatusCode.InternalServerError

ElseIf errValue = 429 And round = 0 Then

Thread.Sleep(savedPostsSleepTimer)

round += 1

Else

_completed = True

End If

End Try

Loop While Not _completed

End Sub

#End Region

#Region "GetUserInfo"

Private Sub GetUserInfo(Optional ByVal Round As Integer = 0)

Try

If Not IsSavedPosts And ChannelInfo Is Nothing Then

Wait429()

Dim r$ = Responser.GetResponse($"https://reddit.com/{IIf(IsChannel, "r", "user")}/{NameTrue}/about.json",, EDP.ReturnValue)

If Not r.IsEmptyString Then

Err429Reset()

Using j As EContainer = JsonDocument.Parse(r)

If Not j Is Nothing AndAlso j.Contains({"data", "subreddit"}) Then

If ID.IsEmptyString Then ID = j.Value({"data"}, "id")

With j({"data", "subreddit"})

UserSiteNameUpdate(.Value("title"))

UserDescriptionUpdate(.Value("public_description"))

Dim dir As SFile = MyFile.CutPath

Dim fileCrFunc As Func(Of String, SFile) = Function(img) CreateFileFromUrl(img)

If DownloadIconBanner Then

SimpleDownloadAvatar(.Value("icon_img"), fileCrFunc)

SimpleDownloadAvatar(.Value("banner_img"), fileCrFunc)

End If

End With

End If

End Using

ElseIf Err429TryAgain And Round < 2 Then

GetUserInfo(Round + 1)

End If

End If

Catch ex As Exception

End Try

End Sub

#End Region

#Region "ParseContainer"

Private Function ParseContainer(ByVal e As EContainer, ByVal PostID As String, ByVal PostDate As String, Optional ByVal UserID As String = Nothing,

Optional ByVal AllowReparse As Boolean = True, Optional ByVal PostText As String = Nothing) As Boolean

If Not e Is Nothing Then

Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)

Dim eCount As Predicate(Of EContainer) = Function(item) item.Count > 0

Dim added As Boolean = True

Dim tmpUrl$ = e.Value("url").IfNullOrEmpty(e.Value({"source"}, "url"))

If Not tmpUrl.IsEmptyString AndAlso tmpUrl.StringContains({$"{SiteRedGifsKey}.com", $"{SiteGfycatKey}.com"}) Then

If SaveToCache Then

tmpUrl = e.Value({"media", "oembed"}, "thumbnail_url")

If Not tmpUrl.IsEmptyString Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

Else

added = False

End If

Else

_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, tmpUrl, PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

End If

ElseIf CreateImgurMedia(tmpUrl, PostID, PostDate, UserID, IsChannel, PostText) Then

_TotalPostsDownloaded += 1

ElseIf DownloadGallery(e, PostID, PostDate, UserID, SaveToCache, PostText) Then

_TotalPostsDownloaded += 1

ElseIf Not If(e({"media"}, "type")?.Value, String.Empty).IsEmptyString Then

With e("media")

Dim t$ = .Item("type").Value

Select Case t

Case "gallery" : If DownloadGallery(.Self, PostID, PostDate,,, PostText) Then _TotalPostsDownloaded += 1 Else added = False

Case "image", "gifvideo"

Dim resolution As Sizes = Nothing

Dim content As Sizes = Nothing

Dim chosenVal$ = String.Empty

ParseResolutions(e("media"), e("preview"), resolution)

If .Contains("content") Then

content = CreateSize(.Self, "content")

If content.HasError Or content.Data.IsEmptyString Then content = Nothing

End If

If UPicType(t) = UTypes.Picture Then

If Not content.Data.IsEmptyString Then

If Not resolution.Data.IsEmptyString Then

If content.Value >= resolution.Value AndAlso TryImage(content.Data) Then

chosenVal = content.Data

Else

chosenVal = resolution.Data

End If

Else

chosenVal = content.Data

End If

Else

chosenVal = resolution.Data

End If

Else

If Not resolution.Data.IsEmptyString Then

chosenVal = resolution.Data

ElseIf Not content.Data.IsEmptyString Then

chosenVal = content.Data

End If

End If

If Not chosenVal.IsEmptyString Then

_TempMediaList.ListAddValue(MediaFromData(UPicType(t), chosenVal, PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

Else

added = False

End If

Case "video"

If UseM3U8 AndAlso .Item("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, .Value("hlsUrl"), PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

ElseIf Not UseM3U8 AndAlso .Item("fallback_url").XmlIfNothingValue("/").ToLower.Contains("mp4") Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, .Value("fallback_url"), PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

Else

added = False

End If

Case Else : added = False

End Select

End With

ElseIf Not If(e({"media", "reddit_video"}, "fallback_url")?.Value, String.Empty).IsEmptyString Then

tmpUrl = e({"media", "reddit_video"}, "fallback_url").Value

If SaveToCache Then

tmpUrl = GetVideoRedditPreview(e)

If Not tmpUrl.IsEmptyString Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, tmpUrl, PostID, PostDate, UserID, False, PostText), LNC)

_TotalPostsDownloaded += 1

Else

added = False

End If

ElseIf UseM3U8 AndAlso Not If(e({"media", "reddit_video"}, "hls_url")?.Value, String.Empty).IsEmptyString Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, e.Value({"media", "reddit_video"}, "hls_url"), PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

Else

_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, tmpUrl, PostID, PostDate, UserID,, PostText), LNC)

_TotalPostsDownloaded += 1

End If

Else

added = False

End If

If Not added Then

If AllowReparse Then

If If(e.ItemF({"crosspost_parent_list", 0})?.Count, 0) > 0 Then

added = ParseContainer(e.ItemF({"crosspost_parent_list", 0}), PostID, PostDate, UserID, True, PostText)

Else

Dim tPostId$ = e.Value(Node_CrosspostParent).IfNullOrEmpty(e.Value(Node_CrosspostParentId)).IfNullOrEmpty(e.Value(Node_CrosspostRootId))

If Not PostID.IsEmptyString Then

For ri% = 0 To 1

Wait429()

Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue)

If Not r.IsEmptyString Then

Err429Reset()

Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)

If j.ListExists Then

With j.ItemF({0, "data", "children", 0, "data"})

If .ListExists Then added = ParseContainer(.Self, PostID, PostDate, UserID, False, PostText)

End With

End If

End Using

Exit For

End If

Next

End If

End If

End If

If Not added Then

Dim node As EContainer = e({"source", "url"})

Dim tmpType As UTypes = UTypes.Undefined

If Not If(node?.Value, String.Empty).IsEmptyString Then

With node.Value.ToLower

Select Case True

Case .Contains(SiteRedGifsKey), .Contains(SiteGfycatKey) : If Not SaveToCache Then tmpType = UTypes.VideoPre

Case .Contains("m3u8") : If Settings.UseM3U8 And Not SaveToCache Then tmpType = UTypes.m3u8

Case .Contains(".gif") And TryFile(node.Value) : tmpType = UTypes.GIF

Case TryFile(node.Value) : tmpType = UTypes.Picture

Case Else : tmpType = UTypes.Undefined

End Select

End With

If Not tmpType = UTypes.Undefined Then

_TempMediaList.ListAddValue(MediaFromData(tmpType, node.Value, PostID, PostDate, UserID,, PostText), LNC)

added = True

End If

End If

If Not added And e.Contains("preview") Then

With e.ItemF({"preview", "images", eCount})

If .ListExists Then

tmpType = UTypes.Undefined

tmpUrl = String.Empty

Dim sv$ = .Value({"source"}, "url")

If Not sv.IsEmptyString AndAlso sv.Contains(".gif") Then

tmpUrl = .Value({"variants", "gif", "source"}, "url")

If Not tmpUrl.IsEmptyString Then tmpType = UTypes.GIF

End If

If tmpUrl.IsEmptyString Then

tmpUrl = .Value({"variants", "mp4", "source"}, "url")

If Not tmpUrl.IsEmptyString Then tmpType = UTypes.Video

End If

If tmpUrl.IsEmptyString Then

tmpUrl = .Value({"source"}, "url")

If Not tmpUrl.IsEmptyString Then tmpType = UTypes.Picture

End If

If Not tmpUrl.IsEmptyString And Not tmpType = UTypes.Undefined Then

Dim m As UserMedia = MediaFromData(tmpType, tmpUrl, PostID, PostDate, UserID,, PostText)

If tmpType = UTypes.Video Then m.File.Extension = "mp4"

_TempMediaList.ListAddValue(m, LNC)

_TotalPostsDownloaded += 1

added = True

End If

End If

End With

End If

End If

End If

If Not added And Not PostText.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(UTypes.Text, String.Empty, PostID, PostDate, UserID,, PostText))

Return added

Else

Return False

End If

End Function

Private Function TryImage(ByVal URL As String) As Boolean

Try

If Not CBool(MySiteSettings.CheckImage.Value) Then

Return MySiteSettings.CheckImageReturnOrig.Value

Else

Dim img As Image = GetImage(SFile.GetBytesFromNet(URL, EDP.ThrowException), EDP.ThrowException)

If Not img Is Nothing Then

img.Dispose()

Return True

Else

Return False

End If

End If

Catch

Return False

End Try

End Function

Private Function GetTextDocument(ByVal e As EContainer) As String

Dim t$ = String.Empty

Try

t = e.Value("title")

With e({"rtjson", "document"})

If .ListExists Then

For Each tt As EContainer In .Self

t.StringAppendLine(vbCrLf,, False)

t.StringAppendLine(If(tt.ItemF({"c", 0, "t"})?.Value, String.Empty))

Next

End If

End With

Catch

End Try

Return t

End Function

#End Region

#Region "Download Base Functions"

Private Function CreateImgurMedia(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,

Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False,

Optional ByVal PostText As String = Nothing) As Boolean

If Not _URL.IsEmptyString AndAlso _URL.Contains("imgur") Then

If _URL.StringContains({".jpg", ".png", ".jpeg"}) Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, _URL, PostID, PostDate, _UserID,, PostText), LNC)

ElseIf _URL.Contains(".gifv") Then

If SaveToCache Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, _URL.Replace(".gifv", ".gif"), PostID, PostDate, _UserID,, PostText), LNC)

Else

_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, _URL.Replace(".gifv", ".mp4"), PostID, PostDate, _UserID,, PostText), LNC)

End If

ElseIf _URL.Contains(".mp4") Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Video, _URL, PostID, PostDate, _UserID,, PostText), LNC)

ElseIf _URL.Contains(".gif") Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.GIF, _URL, PostID, PostDate, _UserID,, PostText), LNC)

Else

Dim obj As IEnumerable(Of UserMedia) = Imgur.Envir.GetVideoInfo(_URL, EDP.ReturnValue)

If Not obj.ListExists Then

If Not TryFile(_URL) Then _URL &= ".jpg"

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, _URL, PostID, PostDate, _UserID,, PostText), LNC)

Else

Dim ut As UTypes

Dim m As UserMedia

For Each data As UserMedia In obj

With data

If Not .URL.IsEmptyString Then

If Not .File.IsEmptyString Then

Select Case .File.Extension

Case "jpg", "png", "jpeg" : ut = UTypes.Picture

Case "gifv" : ut = IIf(SaveToCache, UTypes.Picture, UTypes.Video)

Case "mp4" : ut = UTypes.Video

Case "gif" : ut = UTypes.GIF

Case Else : ut = UTypes.Picture : .File.Extension = "jpg"

End Select

m = MediaFromData(ut, _URL, PostID, PostDate, _UserID,, PostText)

m.URL = .URL

m.File = .File.File

_TempMediaList.ListAddValue(m, LNC)

End If

End If

End With

Next

End If

End If

Return True

Else

Return False

End If

End Function

Private Function DownloadGallery(ByVal e As EContainer, ByVal PostID As String, ByVal PostDate As String,

Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False,

Optional ByVal PostText As String = Nothing) As Boolean

Try

Dim added As Boolean = False

Dim node As EContainer = Nothing

If e.Contains("media_metadata") Then

node = e("media_metadata")

ElseIf e.Contains("mediaMetadata") Then

node = e("mediaMetadata")

End If

If If(node?.Count, 0) > 0 Then

Dim t As EContainer

For Each n As EContainer In node

t = n.ItemF({"s", "u"})

If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then

_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID,, PostText), LNC)

added = True

If FirstOnly Then Exit For

End If

Next

End If

Return added

Catch ex As Exception

ProcessException(ex, Nothing, "gallery parsing error", False)

Return False

End Try

End Function

Private Function GetVideoRedditPreview(ByVal Node As EContainer) As String

Try

If Not Node Is Nothing Then

Dim n As EContainer = Node.ItemF({"preview", "images", 0})

Dim DestNode$() = Nothing

If If(n?.Count, 0) > 0 Then Return ParseResolutions(n)

End If

Return String.Empty

Catch ex As Exception

ProcessException(ex, Nothing, "reddit video preview parsing error", False)

Return String.Empty

End Try

End Function

Private Function ParseResolutions(ByVal Node As EContainer, Optional ByVal PreviewNode As EContainer = Nothing,

Optional ByRef SResult As Sizes = Nothing) As String

Try

If If(Node?.Count, 0) > 0 Then

Dim DestNode$() = Nothing

If If(Node("resolutions")?.Count, 0) > 0 Then

DestNode = {"resolutions"}

ElseIf If(Node({"variants", "nsfw", "resolutions"})?.Count, 0) > 0 Then

DestNode = {"variants", "nsfw", "resolutions"}

End If

If Not DestNode Is Nothing Then

With Node(DestNode)

Dim sl As List(Of Sizes) = .Select(Function(e) CreateSize(e)).

ListWithRemove(Function(ss) ss.HasError Or ss.Data.IsEmptyString)

If If(PreviewNode?.Count, 0) > 0 Then

Dim sp As Sizes = CreateSize(PreviewNode)

If Not sp.HasError And Not sp.Data.IsEmptyString Then

If sl Is Nothing Then sl = New List(Of Sizes)

sl.Add(sp)

End If

End If

If sl.ListExists Then

Dim s As Sizes

sl.Sort()

s = sl.First

sl.Clear()

SResult = s

Return s.Data

End If

End With

End If

End If

Return String.Empty

Catch ex As Exception

Return String.Empty

End Try

End Function

Private Function CreateSize(ByVal Node As EContainer, Optional ByVal UrlNodeName As String = "url") As Sizes

Return New Sizes(Node.Value("width"), Node.Value(UrlNodeName))

End Function

#End Region

#Region "ReparseVideo"

Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)

Dim RedGifsResponser As Responser = Nothing

Try

ThrowAny(Token)

Const v2 As UTypes = UTypes.VideoPre + UTypes.m3u8