认识BOM

什么是BOM

　　BOM（byte-order mark），即字节顺序标记，它是插入到以UTF-8、UTF16或UTF-32编码Unicode文件开头的特殊标记，用来识别Unicode文件的编码类型。对于UTF-8来说，BOM并不是必须的，因为BOM用来标记多字节编码文件的编码类型和字节顺序（big-endian或little-endian）。　


　在绝大多数编辑器中都看不到BOM字符，因为它们能理解Unicode，去掉了读取器看不到的题头信息。若要查看某个Unicode文件是否以BOM开头，可以使用十六进制编辑器。下表列出了不同编码所对应的BOM。


BOM
　　Encoding
　　EF BB BF
　　UTF-8
　　FE FF
　　UTF-16 (big-endian)
　　FF FE
　　UTF-16 (little-endian)
　　00 00 FE FF
　　UTF-32 (big-endian)
　　FF FE 00 00
　　UTF-32 (little-endian)

BOM的来历

　　为了识别 Unicode 文件，Microsoft 建议所有的 Unicode 文件应该以 ZERO WIDTH NOBREAK SPACE（U+FEFF）字符开头。这作为一个“特征符”或“字节顺序标记（byte-order mark，BOM）”来识别文件中使用的编码和字节顺序。

　　不同的系统对BOM的支持

　　因为一些系统或程序不支持BOM，因此带有BOM的Unicode文件有时会带来一些问题。

　　1.　　　　　　　 JDK1.5以及之前的Reader都不能处理带有BOM的UTF-8编码的文件，解析这种格式的xml文件时，会抛出异常：Content is not allowed in prolog.

　　2.　　　　　　　 Linux/UNIX 并没有使用 BOM，因为它会破坏现有的 ASCII 文件的语法约定。

　　不同的编辑工具对BOM的处理也各不相同。使用Windows自带的记事本将文件保存为UTF-8编码的时候，记事本会自动在文件开头插入BOM（虽然BOM对UTF-8来说并不是必须的），但是editplus就不会这样做。

　　BOM与XML

　　XML解析读取XML文档时，W3C定义了3条规则：

　　1.　　　　　　　 如果文档中有BOM，就定义了文件编码；

　　2.　　　　　　　 如果文档中没有BOM，就查看XML声明中的编码属性；

　　3.　　　　　　　 如果上述两者都没有，就假定XML文档采用UTF-8编码。



java处理这类文件可参照老外的博文http://koti.mbnet.fi/akini/java/unicodereader/

主要有个处理类，如下，好东西呀 

UnicodeInputStream.java


/**
version: 1.1 / 2007-01-25
- changed BOM recognition ordering (longer boms first)

Original pseudocode   : Thomas Weidenfeller
Implementation tweaked: Aki Nieminen

http://www.unicode.org/unicode/faq/utf_bom.html
BOMs in byte length ordering:
   00 00 FE FF    = UTF-32, big-endian
   FF FE 00 00    = UTF-32, little-endian
   EF BB BF       = UTF-8,
   FE FF          = UTF-16, big-endian
   FF FE          = UTF-16, little-endian

Win2k Notepad:
   Unicode format = UTF-16LE
***/

import java.io.*;

/**
* This inputstream will recognize unicode BOM marks
* and will skip bytes if getEncoding() method is called
* before any of the read(...) methods.
*
* Usage pattern:
     String enc = "ISO-8859-1"; // or NULL to use systemdefault
     FileInputStream fis = new FileInputStream(file);
     UnicodeInputStream uin = new UnicodeInputStream(fis, enc);
     enc = uin.getEncoding(); // check and skip possible BOM bytes
     InputStreamReader in;
     if (enc == null) in = new InputStreamReader(uin);
     else in = new InputStreamReader(uin, enc);
*/
public class UnicodeInputStream extends InputStream {
   PushbackInputStream internalIn;
   boolean             isInited = false;
String              defaultEnc;
String              encoding;

private static final int BOM_SIZE = 4;

UnicodeInputStream(InputStream in, String defaultEnc) {
internalIn = new PushbackInputStream(in, BOM_SIZE);
this.defaultEnc = defaultEnc;
}

public String getDefaultEncoding() {
      return defaultEnc;
   }

   public String getEncoding() {
      if (!isInited) {
         try {
            init();
         } catch (IOException ex) {
            IllegalStateException ise = new IllegalStateException("Init method failed.");
            ise.initCause(ise);
            throw ise;
         }
      }
      return encoding;
   }

   /**
    * Read-ahead four bytes and check for BOM marks. Extra bytes are
    * unread back to the stream, only BOM bytes are skipped.
    */
   protected void init() throws IOException {
      if (isInited) return;

      byte bom[] = new byte[BOM_SIZE];
      int n, unread;
      n = internalIn.read(bom, 0, bom.length);

      if ( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) &&
                  (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) {
         encoding = "UTF-32BE";
         unread = n - 4;
      } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) &&
                  (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) {
         encoding = "UTF-32LE";
         unread = n - 4;
      } else if (  (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) &&
            (bom[2] == (byte)0xBF) ) {
         encoding = "UTF-8";
         unread = n - 3;
      } else if ( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) {
         encoding = "UTF-16BE";
         unread = n - 2;
      } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) {
         encoding = "UTF-16LE";
         unread = n - 2;
      } else {
         // Unicode BOM mark not found, unread all bytes
         encoding = defaultEnc;
         unread = n;
      }     
      //System.out.println("read=" + n + ", unread=" + unread);

      if (unread > 0) internalIn.unread(bom, (n - unread), unread);

      isInited = true;
   }

   public void close() throws IOException {
      //init();
      isInited = true;
      internalIn.close();
   }

   public int read() throws IOException {
      //init();
      isInited = true;
      return internalIn.read();
   }
}
UnicodeReader.java
/**
version: 1.1 / 2007-01-25
- changed BOM recognition ordering (longer boms first)

Original pseudocode   : Thomas Weidenfeller
Implementation tweaked: Aki Nieminen

http://www.unicode.org/unicode/faq/utf_bom.html
BOMs:
   00 00 FE FF    = UTF-32, big-endian
   FF FE 00 00    = UTF-32, little-endian
   EF BB BF       = UTF-8,
   FE FF          = UTF-16, big-endian
   FF FE          = UTF-16, little-endian

Win2k Notepad:
   Unicode format = UTF-16LE
***/

import java.io.*;

/**
* Generic unicode textreader, which will use BOM mark
* to identify the encoding to be used. If BOM is not found
* then use a given default or system encoding.
*/
public class UnicodeReader extends Reader {
   PushbackInputStream internalIn;
   InputStreamReader   internalIn2 = null;
   String              defaultEnc;

   private static final int BOM_SIZE = 4;

   /**
    *
    * @param in  inputstream to be read
    * @param defaultEnc default encoding if stream does not have
    *                   BOM marker. Give NULL to use system-level default.
    */
   UnicodeReader(InputStream in, String defaultEnc) {
      internalIn = new PushbackInputStream(in, BOM_SIZE);
      this.defaultEnc = defaultEnc;
   }

   public String getDefaultEncoding() {
      return defaultEnc;
   }

   /**
    * Get stream encoding or NULL if stream is uninitialized.
    * Call init() or read() method to initialize it.
    */
   public String getEncoding() {
      if (internalIn2 == null) return null;
      return internalIn2.getEncoding();
   }

   /**
    * Read-ahead four bytes and check for BOM marks. Extra bytes are
    * unread back to the stream, only BOM bytes are skipped.
    */
   protected void init() throws IOException {
      if (internalIn2 != null) return;

      String encoding;
      byte bom[] = new byte[BOM_SIZE];
      int n, unread;
      n = internalIn.read(bom, 0, bom.length);

      if ( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) &&
                  (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) {
         encoding = "UTF-32BE";
         unread = n - 4;
      } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) &&
                  (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) {
         encoding = "UTF-32LE";
         unread = n - 4;
      } else if (  (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) &&
            (bom[2] == (byte)0xBF) ) {
         encoding = "UTF-8";
         unread = n - 3;
      } else if ( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) {
         encoding = "UTF-16BE";
         unread = n - 2;
      } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) {
         encoding = "UTF-16LE";
         unread = n - 2;
      } else {
         // Unicode BOM mark not found, unread all bytes
         encoding = defaultEnc;
         unread = n;
      }   
      //System.out.println("read=" + n + ", unread=" + unread);

      if (unread > 0) internalIn.unread(bom, (n - unread), unread);

      // Use given encoding
      if (encoding == null) {
         internalIn2 = new InputStreamReader(internalIn);
      } else {
         internalIn2 = new InputStreamReader(internalIn, encoding);
      }
   }

   public void close() throws IOException {
      init();
      internalIn2.close();
   }

   public int read(char[] cbuf, int off, int len) throws IOException {
      init();
      return internalIn2.read(cbuf, off, len);
   }

}

 

评论

1 楼 lizhou 2012-08-21  

周sir玩的深啊