You can open the file as if it were a pure text file and check the first six
characters. If it's a real RTF file, those characters will be "{\rtf1".

Sub demo()
   Dim dlg As Dialog
   Dim msg As String

   Set dlg = Dialogs(wdDialogFileOpen)
   With dlg
       If .Display = -1 Then
            msg = .Name & " is"
            If Not IsRtf(WordBasic.filenameinfo$(.Name, 1)) Then
               msg = msg & " not"
            End If
            msg = msg & " a real RTF file"
            MsgBox msg
       End If
   End With
End Sub

Private Function IsRtf(filename As String) As Boolean
   Dim firstLine As String
   IsRtf = False

   Open filename For Input As #1
   Line Input #1, firstLine
   Close #1

   If LCase(Left(firstLine, 6)) = "{\rtf1" Then
       IsRtf = True
   End If
End Function